[cn1:14209] progress: initialized event flag to: 3
[cn1:14209] progress: initialized yield_when_idle to: false
[cn1:14209] progress: initialized num users to: 0
[cn1:14209] progress: initialized poll rate to: 23940000
[cn1:14209] progress: event_users_increment setting count to 1
[cn1:14209] mca: base: components_register: registering ess components
[cn1:14209] mca: base: components_register: found loaded component env
[cn1:14209] mca: base: components_register: component env has no register or open function
[cn1:14209] mca: base: components_register: found loaded component hnp
[cn1:14209] mca: base: components_register: component hnp has no register or open function
[cn1:14209] mca: base: components_register: found loaded component pmi
[cn1:14209] mca: base: components_register: component pmi has no register or open function
[cn1:14209] mca: base: components_register: found loaded component singleton
[cn1:14209] mca: base: components_register: component singleton register function successful
[cn1:14209] mca: base: components_register: found loaded component slurm
[cn1:14209] mca: base: components_register: component slurm has no register or open function
[cn1:14209] mca: base: components_register: found loaded component tool
[cn1:14209] mca: base: components_register: component tool has no register or open function
[cn1:14209] mca: base: components_open: opening ess components
[cn1:14209] mca: base: components_open: found loaded component env
[cn1:14209] mca: base: components_open: component env open function successful
[cn1:14209] mca: base: components_open: found loaded component hnp
[cn1:14209] mca: base: components_open: component hnp open function successful
[cn1:14209] mca: base: components_open: found loaded component pmi
[cn1:14209] mca: base: components_open: component pmi open function successful
[cn1:14209] mca: base: components_open: found loaded component singleton
[cn1:14209] mca: base: components_open: component singleton open function successful
[cn1:14209] mca: base: components_open: found loaded component slurm
[cn1:14209] mca: base: components_open: component slurm open function successful
[cn1:14209] mca: base: components_open: found loaded component tool
[cn1:14209] mca: base: components_open: component tool open function successful
[cn1:14209] mca:base:select: Auto-selecting ess components
[cn1:14209] mca:base:select:(  ess) Querying component [env]
[cn1:14209] mca:base:select:(  ess) Skipping component [env]. Query failed to return a module
[cn1:14209] mca:base:select:(  ess) Querying component [hnp]
[cn1:14209] mca:base:select:(  ess) Query of component [hnp] set priority to 100
[cn1:14209] mca:base:select:(  ess) Querying component [pmi]
[cn1:14209] mca:base:select:(  ess) Skipping component [pmi]. Query failed to return a module
[cn1:14209] mca:base:select:(  ess) Querying component [singleton]
[cn1:14209] mca:base:select:(  ess) Skipping component [singleton]. Query failed to return a module
[cn1:14209] mca:base:select:(  ess) Querying component [slurm]
[cn1:14209] mca:base:select:(  ess) Skipping component [slurm]. Query failed to return a module
[cn1:14209] mca:base:select:(  ess) Querying component [tool]
[cn1:14209] mca:base:select:(  ess) Skipping component [tool]. Query failed to return a module
[cn1:14209] mca:base:select:(  ess) Selected component [hnp]
[cn1:14209] mca: base: close: component env closed
[cn1:14209] mca: base: close: unloading component env
[cn1:14209] mca: base: close: component pmi closed
[cn1:14209] mca: base: close: unloading component pmi
[cn1:14209] mca: base: close: component singleton closed
[cn1:14209] mca: base: close: unloading component singleton
[cn1:14209] mca: base: close: component slurm closed
[cn1:14209] mca: base: close: unloading component slurm
[cn1:14209] mca: base: close: component tool closed
[cn1:14209] mca: base: close: unloading component tool
[cn1:14209] [[INVALID],INVALID] Topology Info:
[cn1:14209] Type: Machine Number of child objects: 2
	Name=NULL
	total=25100516KB
	DMIProductName=S5520UR
	DMIProductVersion=....................
	DMIBoardVendor="Intel Corporation"
	DMIBoardName=S5520UR
	DMIBoardVersion=E22554-751
	DMIBoardAssetTag=....................
	DMIChassisVendor=...............................
	DMIChassisType=23
	DMIChassisVersion=..................
	DMIChassisAssetTag=....................
	DMIBIOSVendor="Intel Corp."
	DMIBIOSVersion=S5500.86B.01.00.0054.092820101104
	DMIBIOSDate=09/28/2010
	DMISysVendor="Intel Corporation"
	Backend=Linux
	LinuxCgroup=/
	OSName=Linux
	OSRelease=2.6.32-431.3.1.el6.x86_64
	OSVersion="#1 SMP Fri Jan 3 21:39:27 UTC 2014"
	Architecture=x86_64
	Cpuset:  0x000000ff
	Online:  0x000000ff
	Allowed: 0x000000ff
	Bind CPU proc:   TRUE
	Bind CPU thread: TRUE
	Bind MEM proc:   FALSE
	Bind MEM thread: FALSE
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12517604KB
		total=12517604KB
		Cpuset:  0x00000055
		Online:  0x00000055
		Allowed: 0x00000055
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x00000055
			Online:  0x00000055
			Allowed: 0x00000055
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x00000055
				Online:  0x00000055
				Allowed: 0x00000055
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000001
					Online:  0x00000001
					Allowed: 0x00000001
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000001
						Online:  0x00000001
						Allowed: 0x00000001
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000001
							Online:  0x00000001
							Allowed: 0x00000001
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000001
								Online:  0x00000001
								Allowed: 0x00000001
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000004
					Online:  0x00000004
					Allowed: 0x00000004
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000004
						Online:  0x00000004
						Allowed: 0x00000004
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000004
							Online:  0x00000004
							Allowed: 0x00000004
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000004
								Online:  0x00000004
								Allowed: 0x00000004
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000010
					Online:  0x00000010
					Allowed: 0x00000010
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000010
						Online:  0x00000010
						Allowed: 0x00000010
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000010
							Online:  0x00000010
							Allowed: 0x00000010
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000010
								Online:  0x00000010
								Allowed: 0x00000010
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000040
					Online:  0x00000040
					Allowed: 0x00000040
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000040
						Online:  0x00000040
						Allowed: 0x00000040
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000040
							Online:  0x00000040
							Allowed: 0x00000040
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000040
								Online:  0x00000040
								Allowed: 0x00000040
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12582912KB
		total=12582912KB
		Cpuset:  0x000000aa
		Online:  0x000000aa
		Allowed: 0x000000aa
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x000000aa
			Online:  0x000000aa
			Allowed: 0x000000aa
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x000000aa
				Online:  0x000000aa
				Allowed: 0x000000aa
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000002
					Online:  0x00000002
					Allowed: 0x00000002
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000002
						Online:  0x00000002
						Allowed: 0x00000002
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000002
							Online:  0x00000002
							Allowed: 0x00000002
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000002
								Online:  0x00000002
								Allowed: 0x00000002
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000008
					Online:  0x00000008
					Allowed: 0x00000008
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000008
						Online:  0x00000008
						Allowed: 0x00000008
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000008
							Online:  0x00000008
							Allowed: 0x00000008
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000008
								Online:  0x00000008
								Allowed: 0x00000008
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000020
					Online:  0x00000020
					Allowed: 0x00000020
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000020
						Online:  0x00000020
						Allowed: 0x00000020
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000020
							Online:  0x00000020
							Allowed: 0x00000020
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000020
								Online:  0x00000020
								Allowed: 0x00000020
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000080
					Online:  0x00000080
					Allowed: 0x00000080
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000080
						Online:  0x00000080
						Allowed: 0x00000080
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000080
							Online:  0x00000080
							Allowed: 0x00000080
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000080
								Online:  0x00000080
								Allowed: 0x00000080
[cn1:14209] mca: base: components_register: registering plm components
[cn1:14209] mca: base: components_register: found loaded component isolated
[cn1:14209] mca: base: components_register: component isolated has no register or open function
[cn1:14209] mca: base: components_register: found loaded component rsh
[cn1:14209] mca: base: components_register: component rsh register function successful
[cn1:14209] mca: base: components_register: found loaded component slurm
[cn1:14209] mca: base: components_register: component slurm register function successful
[cn1:14209] mca: base: components_open: opening plm components
[cn1:14209] mca: base: components_open: found loaded component isolated
[cn1:14209] mca: base: components_open: component isolated open function successful
[cn1:14209] mca: base: components_open: found loaded component rsh
[cn1:14209] mca: base: components_open: component rsh open function successful
[cn1:14209] mca: base: components_open: found loaded component slurm
[cn1:14209] mca: base: components_open: component slurm open function successful
[cn1:14209] mca:base:select: Auto-selecting plm components
[cn1:14209] mca:base:select:(  plm) Querying component [isolated]
[cn1:14209] mca:base:select:(  plm) Query of component [isolated] set priority to 0
[cn1:14209] mca:base:select:(  plm) Querying component [rsh]
[cn1:14209] [[INVALID],INVALID] plm:rsh_lookup on agent ssh : rsh path NULL
[cn1:14209] mca:base:select:(  plm) Query of component [rsh] set priority to 10
[cn1:14209] mca:base:select:(  plm) Querying component [slurm]
[cn1:14209] [[INVALID],INVALID] plm:slurm: available for selection
[cn1:14209] mca:base:select:(  plm) Query of component [slurm] set priority to 75
[cn1:14209] mca:base:select:(  plm) Selected component [slurm]
[cn1:14209] mca: base: close: component isolated closed
[cn1:14209] mca: base: close: unloading component isolated
[cn1:14209] mca: base: close: component rsh closed
[cn1:14209] mca: base: close: unloading component rsh
[cn1:14209] plm:base:set_hnp_name: initial bias 14209 nodename hash 1752458269
[cn1:14209] plm:base:set_hnp_name: final jobfam 15336
[cn1:14209] mca: base: components_register: registering oob components
[cn1:14209] mca: base: components_register: found loaded component tcp
[cn1:14209] mca: base: components_register: component tcp register function successful
[cn1:14209] mca: base: components_open: opening oob components
[cn1:14209] mca: base: components_open: found loaded component tcp
[cn1:14209] mca: base: components_open: component tcp open function successful
[cn1:14209] mca:oob:select: checking available component tcp
[cn1:14209] mca:oob:select: Querying component [tcp]
[cn1:14209] oob:tcp: component_available called
[cn1:14209] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn1:14209] [[15336,0],0] oob:tcp:init rejecting loopback interface lo
[cn1:14209] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn1:14209] [[15336,0],0] oob:tcp:init creating module for V4 address on interface eth0
[cn1:14209] [[15336,0],0] creating OOB-TCP module for interface eth0
[cn1:14209] [[15336,0],0] oob:tcp:init adding 192.168.1.1 to our list of V4 connections
[cn1:14209] WORKING INTERFACE 3 KERNEL INDEX 3 FAMILY: V4
[cn1:14209] [[15336,0],0] oob:tcp:init creating module for V4 address on interface eth1
[cn1:14209] [[15336,0],0] creating OOB-TCP module for interface eth1
[cn1:14209] [[15336,0],0] oob:tcp:init adding 192.168.10.2 to our list of V4 connections
[cn1:14209] [[15336,0],0] TCP STARTUP
[cn1:14209] [[15336,0],0] attempting to bind to IPv4 port 0
[cn1:14209] [[15336,0],0] assigned IPv4 port 37347
[cn1:14209] mca:oob:select: Adding component to end
[cn1:14209] mca:oob:select: Found 1 active transports
[cn1:14209] mca: base: components_register: registering rml components
[cn1:14209] mca: base: components_register: found loaded component oob
[cn1:14209] mca: base: components_register: component oob has no register or open function
[cn1:14209] mca: base: components_open: opening rml components
[cn1:14209] mca: base: components_open: found loaded component oob
[cn1:14209] mca: base: components_open: component oob open function successful
[cn1:14209] orte_rml_base_select: initializing rml component oob
[cn1:14209] mca: base: components_register: registering routed components
[cn1:14209] mca: base: components_register: found loaded component binomial
[cn1:14209] mca: base: components_register: component binomial has no register or open function
[cn1:14209] mca: base: components_register: found loaded component debruijn
[cn1:14209] mca: base: components_register: component debruijn has no register or open function
[cn1:14209] mca: base: components_register: found loaded component direct
[cn1:14209] mca: base: components_register: component direct has no register or open function
[cn1:14209] mca: base: components_register: found loaded component radix
[cn1:14209] mca: base: components_register: component radix register function successful
[cn1:14209] mca: base: components_open: opening routed components
[cn1:14209] mca: base: components_open: found loaded component binomial
[cn1:14209] mca: base: components_open: found loaded component debruijn
[cn1:14209] mca: base: components_open: found loaded component direct
[cn1:14209] mca: base: components_open: found loaded component radix
[cn1:14209] mca:base:select: Auto-selecting routed components
[cn1:14209] mca:base:select:(routed) Querying component [binomial]
[cn1:14209] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn1:14209] mca:base:select:(routed) Querying component [debruijn]
[cn1:14209] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn1:14209] mca:base:select:(routed) Querying component [direct]
[cn1:14209] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn1:14209] mca:base:select:(routed) Querying component [radix]
[cn1:14209] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn1:14209] mca:base:select:(routed) Selected component [binomial]
[cn1:14209] mca: base: close: unloading component debruijn
[cn1:14209] mca: base: close: unloading component direct
[cn1:14209] mca: base: close: unloading component radix
[cn1:14209] orte_routed_base_select: initializing selected component binomial
[cn1:14209] mca: base: components_register: registering dstore components
[cn1:14209] mca: base: components_register: found loaded component hash
[cn1:14209] mca: base: components_register: component hash register function successful
[cn1:14209] mca: base: components_open: opening dstore components
[cn1:14209] mca: base: components_open: found loaded component hash
[cn1:14209] mca:dstore:select: checking available component hash
[cn1:14209] mca:dstore:select: Querying component [hash]
[cn1:14209] mca: base: components_register: registering grpcomm components
[cn1:14209] mca: base: components_register: found loaded component bad
[cn1:14209] mca: base: components_register: component bad has no register or open function
[cn1:14209] mca: base: components_register: found loaded component pmi
[cn1:14209] mca: base: components_register: component pmi register function successful
[cn1:14209] mca: base: components_open: opening grpcomm components
[cn1:14209] mca: base: components_open: found loaded component bad
[cn1:14209] mca: base: components_open: component bad open function successful
[cn1:14209] mca: base: components_open: found loaded component pmi
[cn1:14209] mca: base: components_open: component pmi open function successful
[cn1:14209] mca:base:select: Auto-selecting grpcomm components
[cn1:14209] mca:base:select:(grpcomm) Querying component [bad]
[cn1:14209] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn1:14209] mca:base:select:(grpcomm) Querying component [pmi]
[cn1:14209] mca:base:select:(grpcomm) Skipping component [pmi]. Query failed to return a module
[cn1:14209] mca:base:select:(grpcomm) Selected component [bad]
[cn1:14209] mca: base: close: component pmi closed
[cn1:14209] mca: base: close: unloading component pmi
[cn1:14209] [[15336,0],0] grpcomm:base:receive start comm
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 15
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 32
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 33
[cn1:14209] [[15336,0],0] plm:base:receive start comm
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 5
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 10
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 12
[cn1:14209] mca: base: components_register: registering ras components
[cn1:14209] mca: base: components_register: found loaded component loadleveler
[cn1:14209] mca: base: components_register: component loadleveler register function successful
[cn1:14209] mca: base: components_register: found loaded component simulator
[cn1:14209] mca: base: components_register: component simulator register function successful
[cn1:14209] mca: base: components_register: found loaded component slurm
[cn1:14209] mca: base: components_register: component slurm register function successful
[cn1:14209] mca: base: components_open: opening ras components
[cn1:14209] mca: base: components_open: found loaded component loadleveler
[cn1:14209] mca: base: components_open: component loadleveler open function successful
[cn1:14209] mca: base: components_open: found loaded component simulator
[cn1:14209] mca: base: components_open: found loaded component slurm
[cn1:14209] mca: base: components_open: component slurm open function successful
[cn1:14209] mca:base:select: Auto-selecting ras components
[cn1:14209] mca:base:select:(  ras) Querying component [loadleveler]
[cn1:14209] [[15336,0],0] ras:loadleveler: NOT available for selection
[cn1:14209] mca:base:select:(  ras) Skipping component [loadleveler]. Query failed to return a module
[cn1:14209] mca:base:select:(  ras) Querying component [simulator]
[cn1:14209] mca:base:select:(  ras) Skipping component [simulator]. Query failed to return a module
[cn1:14209] mca:base:select:(  ras) Querying component [slurm]
[cn1:14209] [[15336,0],0] ras:slurm: available for selection
[cn1:14209] mca:base:select:(  ras) Query of component [slurm] set priority to 50
[cn1:14209] mca:base:select:(  ras) Selected component [slurm]
[cn1:14209] mca: base: close: unloading component loadleveler
[cn1:14209] mca: base: close: unloading component simulator
[cn1:14209] mca: base: components_register: registering rmaps components
[cn1:14209] mca: base: components_register: found loaded component lama
[cn1:14209] mca:rmaps:lama: Priority   0
[cn1:14209] mca:rmaps:lama: Map   : NULL
[cn1:14209] mca:rmaps:lama: Bind  : NULL
[cn1:14209] mca:rmaps:lama: MPPR  : NULL
[cn1:14209] mca:rmaps:lama: Order : NULL
[cn1:14209] mca: base: components_register: component lama register function successful
[cn1:14209] mca: base: components_register: found loaded component mindist
[cn1:14209] mca: base: components_register: component mindist register function successful
[cn1:14209] mca: base: components_register: found loaded component ppr
[cn1:14209] mca: base: components_register: component ppr register function successful
[cn1:14209] mca: base: components_register: found loaded component rank_file
[cn1:14209] mca: base: components_register: component rank_file register function successful
[cn1:14209] mca: base: components_register: found loaded component resilient
[cn1:14209] mca: base: components_register: component resilient register function successful
[cn1:14209] mca: base: components_register: found loaded component round_robin
[cn1:14209] mca: base: components_register: component round_robin register function successful
[cn1:14209] mca: base: components_register: found loaded component seq
[cn1:14209] mca: base: components_register: component seq register function successful
[cn1:14209] mca: base: components_register: found loaded component staged
[cn1:14209] mca: base: components_register: component staged has no register or open function
[cn1:14209] mca: base: components_open: opening rmaps components
[cn1:14209] mca: base: components_open: found loaded component lama
[cn1:14209] mca: base: components_open: found loaded component mindist
[cn1:14209] mca: base: components_open: component mindist open function successful
[cn1:14209] mca: base: components_open: found loaded component ppr
[cn1:14209] mca: base: components_open: component ppr open function successful
[cn1:14209] mca: base: components_open: found loaded component rank_file
[cn1:14209] mca: base: components_open: component rank_file open function successful
[cn1:14209] mca: base: components_open: found loaded component resilient
[cn1:14209] mca: base: components_open: component resilient open function successful
[cn1:14209] mca: base: components_open: found loaded component round_robin
[cn1:14209] mca: base: components_open: component round_robin open function successful
[cn1:14209] mca: base: components_open: found loaded component seq
[cn1:14209] mca: base: components_open: component seq open function successful
[cn1:14209] mca: base: components_open: found loaded component staged
[cn1:14209] mca: base: components_open: component staged open function successful
[cn1:14209] mca:rmaps:select: checking available component lama
[cn1:14209] mca:rmaps:select: Querying component [lama]
[cn1:14209] mca:rmaps:select: checking available component mindist
[cn1:14209] mca:rmaps:select: Querying component [mindist]
[cn1:14209] mca:rmaps:select: checking available component ppr
[cn1:14209] mca:rmaps:select: Querying component [ppr]
[cn1:14209] mca:rmaps:select: checking available component rank_file
[cn1:14209] mca:rmaps:select: Querying component [rank_file]
[cn1:14209] mca:rmaps:select: checking available component resilient
[cn1:14209] mca:rmaps:select: Querying component [resilient]
[cn1:14209] mca:rmaps:select: checking available component round_robin
[cn1:14209] mca:rmaps:select: Querying component [round_robin]
[cn1:14209] mca:rmaps:select: checking available component seq
[cn1:14209] mca:rmaps:select: Querying component [seq]
[cn1:14209] mca:rmaps:select: checking available component staged
[cn1:14209] mca:rmaps:select: Querying component [staged]
[cn1:14209] [[15336,0],0]: Final mapper priorities
[cn1:14209] 	Mapper: ppr Priority: 90
[cn1:14209] 	Mapper: seq Priority: 60
[cn1:14209] 	Mapper: resilient Priority: 40
[cn1:14209] 	Mapper: mindist Priority: 20
[cn1:14209] 	Mapper: round_robin Priority: 10
[cn1:14209] 	Mapper: staged Priority: 5
[cn1:14209] 	Mapper: lama Priority: 0
[cn1:14209] 	Mapper: rank_file Priority: 0
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 34
[cn1:14209] [[15336,0],0] setting up session dir with
	tmpdir: UNDEF
	host cn1
[cn1:14209] procdir: /tmp/openmpi-sessions-artpol@cn1_0/15336/0/0
[cn1:14209] jobdir: /tmp/openmpi-sessions-artpol@cn1_0/15336/0
[cn1:14209] top: openmpi-sessions-artpol@cn1_0
[cn1:14209] tmp: /tmp
[cn1:14209] sess_dir_cleanup: job session dir does not exist
[cn1:14209] procdir: /tmp/openmpi-sessions-artpol@cn1_0/15336/0/0
[cn1:14209] jobdir: /tmp/openmpi-sessions-artpol@cn1_0/15336/0
[cn1:14209] top: openmpi-sessions-artpol@cn1_0
[cn1:14209] tmp: /tmp
[cn1:14209] [[15336,0],0] writing contact file /tmp/openmpi-sessions-artpol@cn1_0/15336/contact.txt
[cn1:14209] [[15336,0],0] wrote contact file
[cn1:14209] [[15336,0],0] routed_binomial: init routes for HNP job [15336,0]
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 2
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 21
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 22
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 45
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 46
[cn1:14209] progress: progress_set_yield_when_idle to false
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 1
[cn1:14209] [[15336,0],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 27
[cn1:14209] [[15336,0],0] Monitoring debugger attach fifo /tmp/openmpi-sessions-artpol@cn1_0/15336/0/debugger_attach_fifo
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 15 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 32 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 33 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 5 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 10 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 12 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 34 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 2 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 21 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 22 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 45 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 46 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 1 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] posting recv
[cn1:14209] [[15336,0],0] posting persistent recv on tag 27 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] plm:base:setup_job
[cn1:14209] [[15336,0],0] ras:base:allocate
[cn1:14209] [[15336,0],0] ras:slurm:allocate:discover: checking nodelist: cn[1-5]
[cn1:14209] [[15336,0],0] ras:slurm:allocate:discover: parse range 1-5 (2)
[cn1:14209] [[15336,0],0] ras:slurm:allocate:discover: adding node cn1 (2 slots)
[cn1:14209] [[15336,0],0] ras:slurm:allocate:discover: adding node cn2 (2 slots)
[cn1:14209] [[15336,0],0] ras:slurm:allocate:discover: adding node cn3 (2 slots)
[cn1:14209] [[15336,0],0] ras:slurm:allocate:discover: adding node cn4 (2 slots)
[cn1:14209] [[15336,0],0] ras:slurm:allocate:discover: adding node cn5 (2 slots)
[cn1:14209] [[15336,0],0] ras:slurm:allocate: success
[cn1:14209] [[15336,0],0] ras:base:node_insert inserting 5 nodes
[cn1:14209] [[15336,0],0] ras:base:node_insert updating HNP [cn1] info to 2 slots
[cn1:14209] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn1:14209] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn1:14209] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn1:14209] [[15336,0],0] ras:base:node_insert node cn2 slots 2
[cn1:14209] [[15336,0],0] ras:base:node_insert node cn3 slots 2
[cn1:14209] [[15336,0],0] ras:base:node_insert node cn4 slots 2
[cn1:14209] [[15336,0],0] ras:base:node_insert node cn5 slots 2
[cn1:14209] [[15336,0],0] plm:slurm: LAUNCH DAEMONS CALLED
[cn1:14209] [[15336,0],0] plm:base:setup_vm
[cn1:14209] [[15336,0],0] plm:base:setup_vm creating map
[cn1:14209] [[15336,0],0] plm:base:setup_vm add new daemon [[15336,0],1]
[cn1:14209] [[15336,0],0] plm:base:setup_vm assigning new daemon [[15336,0],1] to node cn2
[cn1:14209] [[15336,0],0] plm:base:setup_vm add new daemon [[15336,0],2]
[cn1:14209] [[15336,0],0] plm:base:setup_vm assigning new daemon [[15336,0],2] to node cn3
[cn1:14209] [[15336,0],0] plm:base:setup_vm add new daemon [[15336,0],3]
[cn1:14209] [[15336,0],0] plm:base:setup_vm assigning new daemon [[15336,0],3] to node cn4
[cn1:14209] [[15336,0],0] plm:base:setup_vm add new daemon [[15336,0],4]
[cn1:14209] [[15336,0],0] plm:base:setup_vm assigning new daemon [[15336,0],4] to node cn5
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 0 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial 0 found child 1
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 1 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 1 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial 1 found child 3
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 3 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 3 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 1
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 3
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 3 parent 1 me 3 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 1
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 1
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 0
[cn1:14209] [[15336,0],0] routed:binomial 0 found child 2
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 1
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 3
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 3 parent 1 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 3
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 7
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 5
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 2
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 2 parent 0 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 0
[cn1:14209] [[15336,0],0] routed:binomial 0 found child 4
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 1
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 3
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 3 parent 1 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 3
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 7
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 5
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 2
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 2 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 2
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 6
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 4
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 4 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 0
[cn1:14209] [[15336,0],0]: parent 0 num_children 3
[cn1:14209] [[15336,0],0]: 	child 1
[cn1:14209] [[15336,0],0]: 		relation 3
[cn1:14209] [[15336,0],0]: 	child 2
[cn1:14209] [[15336,0],0]: 	child 4
[cn1:14209] [[15336,0],0] plm:slurm: launching on nodes cn2,cn3,cn4,cn5
[cn1:14209] [[15336,0],0] plm:slurm: final top-level argv:
	srun --ntasks-per-node=1 --kill-on-bad-exit --cpu_bind=none --nodes=4 --nodelist=cn2,cn3,cn4,cn5 --ntasks=4 orted --debug --debug-daemons -mca orte_ess_jobid 1005060096 -mca orte_ess_vpid 1 -mca orte_ess_num_procs 5 -mca orte_hnp_uri "1005060096.0;tcp://192.168.1.1,192.168.10.2:37347"
Daemon was launched on cn2 - beginning to initialize
Daemon was launched on cn5 - beginning to initialize
Daemon was launched on cn3 - beginning to initialize
Daemon was launched on cn4 - beginning to initialize
[cn2:01042] progress: initialized event flag to: 3
[cn2:01042] progress: initialized yield_when_idle to: false
[cn2:01042] progress: initialized num users to: 0
[cn2:01042] progress: initialized poll rate to: 23940000
[cn2:01042] progress: event_users_increment setting count to 1
[cn2:01042] mca: base: components_register: registering ess components
[cn2:01042] mca: base: components_register: found loaded component env
[cn2:01042] mca: base: components_register: component env has no register or open function
[cn2:01042] mca: base: components_register: found loaded component hnp
[cn2:01042] mca: base: components_register: component hnp has no register or open function
[cn2:01042] mca: base: components_register: found loaded component pmi
[cn2:01042] mca: base: components_register: component pmi has no register or open function
[cn2:01042] mca: base: components_register: found loaded component singleton
[cn2:01042] mca: base: components_register: component singleton register function successful
[cn2:01042] mca: base: components_register: found loaded component slurm
[cn2:01042] mca: base: components_register: component slurm has no register or open function
[cn2:01042] mca: base: components_register: found loaded component tool
[cn2:01042] mca: base: components_register: component tool has no register or open function
[cn2:01042] mca: base: components_open: opening ess components
[cn2:01042] mca: base: components_open: found loaded component env
[cn2:01042] mca: base: components_open: component env open function successful
[cn2:01042] mca: base: components_open: found loaded component hnp
[cn2:01042] mca: base: components_open: component hnp open function successful
[cn2:01042] mca: base: components_open: found loaded component pmi
[cn2:01042] mca: base: components_open: component pmi open function successful
[cn2:01042] mca: base: components_open: found loaded component singleton
[cn2:01042] mca: base: components_open: component singleton open function successful
[cn2:01042] mca: base: components_open: found loaded component slurm
[cn2:01042] mca: base: components_open: component slurm open function successful
[cn2:01042] mca: base: components_open: found loaded component tool
[cn2:01042] mca: base: components_open: component tool open function successful
[cn2:01042] mca:base:select: Auto-selecting ess components
[cn2:01042] mca:base:select:(  ess) Querying component [env]
[cn2:01042] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn2:01042] mca:base:select:(  ess) Querying component [hnp]
[cn2:01042] mca:base:select:(  ess) Skipping component [hnp]. Query failed to return a module
[cn2:01042] mca:base:select:(  ess) Querying component [pmi]
[cn2:01042] mca:base:select:(  ess) Query of component [pmi] set priority to 35
[cn2:01042] mca:base:select:(  ess) Querying component [singleton]
[cn2:01042] mca:base:select:(  ess) Skipping component [singleton]. Query failed to return a module
[cn2:01042] mca:base:select:(  ess) Querying component [slurm]
[cn2:01042] mca:base:select:(  ess) Query of component [slurm] set priority to 50
[cn2:01042] mca:base:select:(  ess) Querying component [tool]
[cn2:01042] mca:base:select:(  ess) Skipping component [tool]. Query failed to return a module
[cn2:01042] mca:base:select:(  ess) Selected component [slurm]
[cn2:01042] mca: base: close: component env closed
[cn2:01042] mca: base: close: unloading component env
[cn2:01042] mca: base: close: component hnp closed
[cn2:01042] mca: base: close: unloading component hnp
[cn2:01042] mca: base: close: component pmi closed
[cn2:01042] mca: base: close: unloading component pmi
[cn2:01042] mca: base: close: component singleton closed
[cn2:01042] mca: base: close: unloading component singleton
[cn2:01042] mca: base: close: component tool closed
[cn2:01042] mca: base: close: unloading component tool
[cn5:14171] progress: initialized event flag to: 3
[cn5:14171] progress: initialized yield_when_idle to: false
[cn5:14171] progress: initialized num users to: 0
[cn5:14171] progress: initialized poll rate to: 23940000
[cn5:14171] progress: event_users_increment setting count to 1
[cn5:14171] mca: base: components_register: registering ess components
[cn5:14171] mca: base: components_register: found loaded component env
[cn5:14171] mca: base: components_register: component env has no register or open function
[cn5:14171] mca: base: components_register: found loaded component hnp
[cn5:14171] mca: base: components_register: component hnp has no register or open function
[cn5:14171] mca: base: components_register: found loaded component pmi
[cn5:14171] mca: base: components_register: component pmi has no register or open function
[cn5:14171] mca: base: components_register: found loaded component singleton
[cn5:14171] mca: base: components_register: component singleton register function successful
[cn5:14171] mca: base: components_register: found loaded component slurm
[cn5:14171] mca: base: components_register: component slurm has no register or open function
[cn5:14171] mca: base: components_register: found loaded component tool
[cn5:14171] mca: base: components_register: component tool has no register or open function
[cn5:14171] mca: base: components_open: opening ess components
[cn5:14171] mca: base: components_open: found loaded component env
[cn5:14171] mca: base: components_open: component env open function successful
[cn5:14171] mca: base: components_open: found loaded component hnp
[cn5:14171] mca: base: components_open: component hnp open function successful
[cn5:14171] mca: base: components_open: found loaded component pmi
[cn5:14171] mca: base: components_open: component pmi open function successful
[cn5:14171] mca: base: components_open: found loaded component singleton
[cn5:14171] mca: base: components_open: component singleton open function successful
[cn5:14171] mca: base: components_open: found loaded component slurm
[cn5:14171] mca: base: components_open: component slurm open function successful
[cn5:14171] mca: base: components_open: found loaded component tool
[cn5:14171] mca: base: components_open: component tool open function successful
[cn5:14171] mca:base:select: Auto-selecting ess components
[cn5:14171] mca:base:select:(  ess) Querying component [env]
[cn5:14171] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn5:14171] mca:base:select:(  ess) Querying component [hnp]
[cn5:14171] mca:base:select:(  ess) Skipping component [hnp]. Query failed to return a module
[cn5:14171] mca:base:select:(  ess) Querying component [pmi]
[cn5:14171] mca:base:select:(  ess) Query of component [pmi] set priority to 35
[cn5:14171] mca:base:select:(  ess) Querying component [singleton]
[cn5:14171] mca:base:select:(  ess) Skipping component [singleton]. Query failed to return a module
[cn5:14171] mca:base:select:(  ess) Querying component [slurm]
[cn5:14171] mca:base:select:(  ess) Query of component [slurm] set priority to 50
[cn5:14171] mca:base:select:(  ess) Querying component [tool]
[cn5:14171] mca:base:select:(  ess) Skipping component [tool]. Query failed to return a module
[cn5:14171] mca:base:select:(  ess) Selected component [slurm]
[cn5:14171] mca: base: close: component env closed
[cn5:14171] mca: base: close: unloading component env
[cn5:14171] mca: base: close: component hnp closed
[cn5:14171] mca: base: close: unloading component hnp
[cn5:14171] mca: base: close: component pmi closed
[cn5:14171] mca: base: close: unloading component pmi
[cn5:14171] mca: base: close: component singleton closed
[cn5:14171] mca: base: close: unloading component singleton
[cn5:14171] mca: base: close: component tool closed
[cn5:14171] mca: base: close: unloading component tool
[cn2:01042] ess:slurm setting name
[cn2:01042] ess:slurm set name to [[15336,0],1]
[cn2:01042] ess:slurm set nodename to cn2
[cn2:01042] [[15336,0],1] Topology Info:
[cn2:01042] Type: Machine Number of child objects: 2
	Name=NULL
	total=25100516KB
	DMIProductName=S5520UR
	DMIProductVersion=....................
	DMIBoardVendor="Intel Corporation"
	DMIBoardName=S5520UR
	DMIBoardVersion=E22554-751
	DMIBoardAssetTag=....................
	DMIChassisVendor=...............................
	DMIChassisType=23
	DMIChassisVersion=..................
	DMIChassisAssetTag=....................
	DMIBIOSVendor="Intel Corp."
	DMIBIOSVersion=S5500.86B.01.00.0054.092820101104
	DMIBIOSDate=09/28/2010
	DMISysVendor="Intel Corporation"
	Backend=Linux
	LinuxCgroup=/
	OSName=Linux
	OSRelease=2.6.32-431.3.1.el6.x86_64
	OSVersion="#1 SMP Fri Jan 3 21:39:27 UTC 2014"
	Architecture=x86_64
	Cpuset:  0x000000ff
	Online:  0x000000ff
	Allowed: 0x000000ff
	Bind CPU proc:   TRUE
	Bind CPU thread: TRUE
	Bind MEM proc:   FALSE
	Bind MEM thread: FALSE
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12517604KB
		total=12517604KB
		Cpuset:  0x00000055
		Online:  0x00000055
		Allowed: 0x00000055
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x00000055
			Online:  0x00000055
			Allowed: 0x00000055
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x00000055
				Online:  0x00000055
				Allowed: 0x00000055
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000001
					Online:  0x00000001
					Allowed: 0x00000001
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000001
						Online:  0x00000001
						Allowed: 0x00000001
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000001
							Online:  0x00000001
							Allowed: 0x00000001
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000001
								Online:  0x00000001
								Allowed: 0x00000001
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000004
					Online:  0x00000004
					Allowed: 0x00000004
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000004
						Online:  0x00000004
						Allowed: 0x00000004
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000004
							Online:  0x00000004
							Allowed: 0x00000004
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000004
								Online:  0x00000004
								Allowed: 0x00000004
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000010
					Online:  0x00000010
					Allowed: 0x00000010
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000010
						Online:  0x00000010
						Allowed: 0x00000010
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000010
							Online:  0x00000010
							Allowed: 0x00000010
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000010
								Online:  0x00000010
								Allowed: 0x00000010
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000040
					Online:  0x00000040
					Allowed: 0x00000040
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000040
						Online:  0x00000040
						Allowed: 0x00000040
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000040
							Online:  0x00000040
							Allowed: 0x00000040
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000040
								Online:  0x00000040
								Allowed: 0x00000040
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12582912KB
		total=12582912KB
		Cpuset:  0x000000aa
		Online:  0x000000aa
		Allowed: 0x000000aa
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x000000aa
			Online:  0x000000aa
			Allowed: 0x000000aa
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x000000aa
				Online:  0x000000aa
				Allowed: 0x000000aa
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000002
					Online:  0x00000002
					Allowed: 0x00000002
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000002
						Online:  0x00000002
						Allowed: 0x00000002
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000002
							Online:  0x00000002
							Allowed: 0x00000002
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000002
								Online:  0x00000002
								Allowed: 0x00000002
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000008
					Online:  0x00000008
					Allowed: 0x00000008
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000008
						Online:  0x00000008
						Allowed: 0x00000008
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000008
							Online:  0x00000008
							Allowed: 0x00000008
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000008
								Online:  0x00000008
								Allowed: 0x00000008
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000020
					Online:  0x00000020
					Allowed: 0x00000020
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000020
						Online:  0x00000020
						Allowed: 0x00000020
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000020
							Online:  0x00000020
							Allowed: 0x00000020
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000020
								Online:  0x00000020
								Allowed: 0x00000020
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000080
					Online:  0x00000080
					Allowed: 0x00000080
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000080
						Online:  0x00000080
						Allowed: 0x00000080
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000080
							Online:  0x00000080
							Allowed: 0x00000080
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000080
								Online:  0x00000080
								Allowed: 0x00000080
[cn4:17828] progress: initialized event flag to: 3
[cn4:17828] progress: initialized yield_when_idle to: false
[cn4:17828] progress: initialized num users to: 0
[cn4:17828] progress: initialized poll rate to: 23940000
[cn4:17828] progress: event_users_increment setting count to 1
[cn3:21416] progress: initialized event flag to: 3
[cn3:21416] progress: initialized yield_when_idle to: false
[cn3:21416] progress: initialized num users to: 0
[cn3:21416] progress: initialized poll rate to: 23940000
[cn3:21416] progress: event_users_increment setting count to 1
[cn4:17828] mca: base: components_register: registering ess components
[cn4:17828] mca: base: components_register: found loaded component env
[cn4:17828] mca: base: components_register: component env has no register or open function
[cn4:17828] mca: base: components_register: found loaded component hnp
[cn4:17828] mca: base: components_register: component hnp has no register or open function
[cn4:17828] mca: base: components_register: found loaded component pmi
[cn4:17828] mca: base: components_register: component pmi has no register or open function
[cn3:21416] mca: base: components_register: registering ess components
[cn3:21416] mca: base: components_register: found loaded component env
[cn3:21416] mca: base: components_register: component env has no register or open function
[cn3:21416] mca: base: components_register: found loaded component hnp
[cn3:21416] mca: base: components_register: component hnp has no register or open function
[cn3:21416] mca: base: components_register: found loaded component pmi
[cn3:21416] mca: base: components_register: component pmi has no register or open function
[cn4:17828] mca: base: components_register: found loaded component singleton
[cn3:21416] mca: base: components_register: found loaded component singleton
[cn4:17828] mca: base: components_register: component singleton register function successful
[cn3:21416] mca: base: components_register: component singleton register function successful
[cn4:17828] mca: base: components_register: found loaded component slurm
[cn4:17828] mca: base: components_register: component slurm has no register or open function
[cn3:21416] mca: base: components_register: found loaded component slurm
[cn3:21416] mca: base: components_register: component slurm has no register or open function
[cn4:17828] mca: base: components_register: found loaded component tool
[cn4:17828] mca: base: components_register: component tool has no register or open function
[cn3:21416] mca: base: components_register: found loaded component tool
[cn3:21416] mca: base: components_register: component tool has no register or open function
[cn4:17828] mca: base: components_open: opening ess components
[cn4:17828] mca: base: components_open: found loaded component env
[cn4:17828] mca: base: components_open: component env open function successful
[cn4:17828] mca: base: components_open: found loaded component hnp
[cn4:17828] mca: base: components_open: component hnp open function successful
[cn4:17828] mca: base: components_open: found loaded component pmi
[cn3:21416] mca: base: components_open: opening ess components
[cn3:21416] mca: base: components_open: found loaded component env
[cn3:21416] mca: base: components_open: component env open function successful
[cn3:21416] mca: base: components_open: found loaded component hnp
[cn4:17828] mca: base: components_open: component pmi open function successful
[cn4:17828] mca: base: components_open: found loaded component singleton
[cn4:17828] mca: base: components_open: component singleton open function successful
[cn4:17828] mca: base: components_open: found loaded component slurm
[cn4:17828] mca: base: components_open: component slurm open function successful
[cn4:17828] mca: base: components_open: found loaded component tool
[cn4:17828] mca: base: components_open: component tool open function successful
[cn4:17828] mca:base:select: Auto-selecting ess components
[cn4:17828] mca:base:select:(  ess) Querying component [env]
[cn4:17828] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn3:21416] mca: base: components_open: component hnp open function successful
[cn3:21416] mca: base: components_open: found loaded component pmi
[cn3:21416] mca: base: components_open: component pmi open function successful
[cn3:21416] mca: base: components_open: found loaded component singleton
[cn3:21416] mca: base: components_open: component singleton open function successful
[cn3:21416] mca: base: components_open: found loaded component slurm
[cn3:21416] mca: base: components_open: component slurm open function successful
[cn3:21416] mca: base: components_open: found loaded component tool
[cn3:21416] mca: base: components_open: component tool open function successful
[cn3:21416] mca:base:select: Auto-selecting ess components
[cn4:17828] mca:base:select:(  ess) Querying component [hnp]
[cn4:17828] mca:base:select:(  ess) Skipping component [hnp]. Query failed to return a module
[cn4:17828] mca:base:select:(  ess) Querying component [pmi]
[cn3:21416] mca:base:select:(  ess) Querying component [env]
[cn3:21416] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn3:21416] mca:base:select:(  ess) Querying component [hnp]
[cn3:21416] mca:base:select:(  ess) Skipping component [hnp]. Query failed to return a module
[cn3:21416] mca:base:select:(  ess) Querying component [pmi]
[cn4:17828] mca:base:select:(  ess) Query of component [pmi] set priority to 35
[cn4:17828] mca:base:select:(  ess) Querying component [singleton]
[cn4:17828] mca:base:select:(  ess) Skipping component [singleton]. Query failed to return a module
[cn4:17828] mca:base:select:(  ess) Querying component [slurm]
[cn4:17828] mca:base:select:(  ess) Query of component [slurm] set priority to 50
[cn4:17828] mca:base:select:(  ess) Querying component [tool]
[cn4:17828] mca:base:select:(  ess) Skipping component [tool]. Query failed to return a module
[cn4:17828] mca:base:select:(  ess) Selected component [slurm]
[cn4:17828] mca: base: close: component env closed
[cn4:17828] mca: base: close: unloading component env
[cn4:17828] mca: base: close: component hnp closed
[cn4:17828] mca: base: close: unloading component hnp
[cn3:21416] mca:base:select:(  ess) Query of component [pmi] set priority to 35
[cn3:21416] mca:base:select:(  ess) Querying component [singleton]
[cn4:17828] mca: base: close: component pmi closed
[cn4:17828] mca: base: close: unloading component pmi
[cn3:21416] mca:base:select:(  ess) Skipping component [singleton]. Query failed to return a module
[cn3:21416] mca:base:select:(  ess) Querying component [slurm]
[cn3:21416] mca:base:select:(  ess) Query of component [slurm] set priority to 50
[cn3:21416] mca:base:select:(  ess) Querying component [tool]
[cn3:21416] mca:base:select:(  ess) Skipping component [tool]. Query failed to return a module
[cn4:17828] mca: base: close: component singleton closed
[cn4:17828] mca: base: close: unloading component singleton
[cn3:21416] mca:base:select:(  ess) Selected component [slurm]
[cn3:21416] mca: base: close: component env closed
[cn3:21416] mca: base: close: unloading component env
[cn4:17828] mca: base: close: component tool closed
[cn4:17828] mca: base: close: unloading component tool
[cn3:21416] mca: base: close: component hnp closed
[cn3:21416] mca: base: close: unloading component hnp
[cn3:21416] mca: base: close: component pmi closed
[cn3:21416] mca: base: close: unloading component pmi
[cn3:21416] mca: base: close: component singleton closed
[cn3:21416] mca: base: close: unloading component singleton
[cn3:21416] mca: base: close: component tool closed
[cn3:21416] mca: base: close: unloading component tool
[cn3:21416] ess:slurm setting name
[cn3:21416] ess:slurm set name to [[15336,0],2]
[cn3:21416] ess:slurm set nodename to cn3
[cn5:14171] ess:slurm setting name
[cn5:14171] ess:slurm set name to [[15336,0],4]
[cn5:14171] ess:slurm set nodename to cn5
[cn5:14171] [[15336,0],4] Topology Info:
[cn5:14171] Type: Machine Number of child objects: 2
	Name=NULL
	total=25100444KB
	DMIProductName=S5520UR
	DMIProductVersion=....................
	DMIBoardVendor="Intel Corporation"
	DMIBoardName=S5520UR
	DMIBoardVersion=E22554-751
	DMIBoardAssetTag=....................
	DMIChassisVendor=...............................
	DMIChassisType=23
	DMIChassisVersion=..................
	DMIChassisAssetTag=....................
	DMIBIOSVendor="Intel Corp."
	DMIBIOSVersion=S5500.86B.01.00.0054.092820101104
	DMIBIOSDate=09/28/2010
	DMISysVendor="Intel Corporation"
	Backend=Linux
	LinuxCgroup=/
	OSName=Linux
	OSRelease=2.6.32-431.3.1.el6.x86_64
	OSVersion="#1 SMP Fri Jan 3 21:39:27 UTC 2014"
	Architecture=x86_64
	Cpuset:  0x000000ff
	Online:  0x000000ff
	Allowed: 0x000000ff
	Bind CPU proc:   TRUE
	Bind CPU thread: TRUE
	Bind MEM proc:   FALSE
	Bind MEM thread: FALSE
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12517532KB
		total=12517532KB
		Cpuset:  0x00000055
		Online:  0x00000055
		Allowed: 0x00000055
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x00000055
			Online:  0x00000055
			Allowed: 0x00000055
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x00000055
				Online:  0x00000055
				Allowed: 0x00000055
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000001
					Online:  0x00000001
					Allowed: 0x00000001
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000001
						Online:  0x00000001
						Allowed: 0x00000001
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000001
							Online:  0x00000001
							Allowed: 0x00000001
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000001
								Online:  0x00000001
								Allowed: 0x00000001
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000004
					Online:  0x00000004
					Allowed: 0x00000004
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000004
						Online:  0x00000004
						Allowed: 0x00000004
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000004
							Online:  0x00000004
							Allowed: 0x00000004
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000004
								Online:  0x00000004
								Allowed: 0x00000004
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000010
					Online:  0x00000010
					Allowed: 0x00000010
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000010
						Online:  0x00000010
						Allowed: 0x00000010
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000010
							Online:  0x00000010
							Allowed: 0x00000010
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000010
								Online:  0x00000010
								Allowed: 0x00000010
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000040
					Online:  0x00000040
					Allowed: 0x00000040
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000040
						Online:  0x00000040
						Allowed: 0x00000040
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000040
							Online:  0x00000040
							Allowed: 0x00000040
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000040
								Online:  0x00000040
								Allowed: 0x00000040
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12582912KB
		total=12582912KB
		Cpuset:  0x000000aa
		Online:  0x000000aa
		Allowed: 0x000000aa
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x000000aa
			Online:  0x000000aa
			Allowed: 0x000000aa
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x000000aa
				Online:  0x000000aa
				Allowed: 0x000000aa
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000002
					Online:  0x00000002
					Allowed: 0x00000002
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000002
						Online:  0x00000002
						Allowed: 0x00000002
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000002
							Online:  0x00000002
							Allowed: 0x00000002
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000002
								Online:  0x00000002
								Allowed: 0x00000002
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000008
					Online:  0x00000008
					Allowed: 0x00000008
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000008
						Online:  0x00000008
						Allowed: 0x00000008
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000008
							Online:  0x00000008
							Allowed: 0x00000008
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000008
								Online:  0x00000008
								Allowed: 0x00000008
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000020
					Online:  0x00000020
					Allowed: 0x00000020
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000020
						Online:  0x00000020
						Allowed: 0x00000020
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000020
							Online:  0x00000020
							Allowed: 0x00000020
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000020
								Online:  0x00000020
								Allowed: 0x00000020
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000080
					Online:  0x00000080
					Allowed: 0x00000080
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000080
						Online:  0x00000080
						Allowed: 0x00000080
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000080
							Online:  0x00000080
							Allowed: 0x00000080
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000080
								Online:  0x00000080
								Allowed: 0x00000080
[cn2:01042] mca: base: components_register: registering oob components
[cn2:01042] mca: base: components_register: found loaded component tcp
[cn3:21416] [[15336,0],2] Topology Info:
[cn2:01042] mca: base: components_register: component tcp register function successful
[cn3:21416] Type: Machine Number of child objects: 2
	Name=NULL
	total=25100444KB
	DMIProductName=S5520UR
	DMIProductVersion=....................
	DMIBoardVendor="Intel Corporation"
	DMIBoardName=S5520UR
	DMIBoardVersion=E22554-751
	DMIBoardAssetTag=....................
	DMIChassisVendor=...............................
	DMIChassisType=23
	DMIChassisVersion=..................
	DMIChassisAssetTag=....................
	DMIBIOSVendor="Intel Corp."
	DMIBIOSVersion=S5500.86B.01.00.0054.092820101104
	DMIBIOSDate=09/28/2010
	DMISysVendor="Intel Corporation"
	Backend=Linux
	LinuxCgroup=/
	OSName=Linux
	OSRelease=2.6.32-431.3.1.el6.x86_64
	OSVersion="#1 SMP Fri Jan 3 21:39:27 UTC 2014"
	Architecture=x86_64
	Cpuset:  0x000000ff
	Online:  0x000000ff
	Allowed: 0x000000ff
	Bind CPU proc:   TRUE
	Bind CPU thread: TRUE
	Bind MEM proc:   FALSE
	Bind MEM thread: FALSE
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12517532KB
		total=12517532KB
		Cpuset:  0x00000055
		Online:  0x00000055
		Allowed: 0x00000055
[cn2:01042] mca: base: components_open: opening oob components
[cn2:01042] mca: base: components_open: found loaded component tcp
[cn4:17828] ess:slurm setting name
[cn4:17828] ess:slurm set name to [[15336,0],3]
[cn4:17828] ess:slurm set nodename to cn4
[cn2:01042] mca: base: components_open: component tcp open function successful
[cn2:01042] mca:oob:select: checking available component tcp
[cn2:01042] mca:oob:select: Querying component [tcp]
[cn2:01042] oob:tcp: component_available called
[cn4:17828] [[15336,0],3] Topology Info:
[cn2:01042] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn2:01042] [[15336,0],1] oob:tcp:init rejecting loopback interface lo
[cn2:01042] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn2:01042] [[15336,0],1] oob:tcp:init creating module for V4 address on interface eth0
[cn2:01042] [[15336,0],1] creating OOB-TCP module for interface eth0
[cn2:01042] [[15336,0],1] oob:tcp:init adding 192.168.1.2 to our list of V4 connections
[cn2:01042] [[15336,0],1] TCP STARTUP
[cn2:01042] [[15336,0],1] attempting to bind to IPv4 port 0
[cn4:17828] Type: Machine Number of child objects: 2
	Name=NULL
	total=25100516KB
	DMIProductName=S5520UR
	DMIProductVersion=....................
	DMIBoardVendor="Intel Corporation"
	DMIBoardName=S5520UR
	DMIBoardVersion=E22554-751
	DMIBoardAssetTag=....................
	DMIChassisVendor=...............................
	DMIChassisType=23
	DMIChassisVersion=..................
	DMIChassisAssetTag=....................
	DMIBIOSVendor="Intel Corp."
	DMIBIOSVersion=S5500.86B.01.00.0054.092820101104
	DMIBIOSDate=09/28/2010
	DMISysVendor="Intel Corporation"
	Backend=Linux
	LinuxCgroup=/
	OSName=Linux
	OSRelease=2.6.32-431.3.1.el6.x86_64
	OSVersion="#1 SMP Fri Jan 3 21:39:27 UTC 2014"
	Architecture=x86_64
	Cpuset:  0x000000ff
	Online:  0x000000ff
	Allowed: 0x000000ff
	Bind CPU proc:   TRUE
	Bind CPU thread: TRUE
	Bind MEM proc:   FALSE
	Bind MEM thread: FALSE
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12517604KB
		total=12517604KB
		Cpuset:  0x00000055
		Online:  0x00000055
		Allowed: 0x00000055
[cn2:01042] [[15336,0],1] assigned IPv4 port 51705
[cn2:01042] mca:oob:select: Adding component to end
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x00000055
			Online:  0x00000055
			Allowed: 0x00000055
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x00000055
				Online:  0x00000055
				Allowed: 0x00000055
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000001
					Online:  0x00000001
					Allowed: 0x00000001
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000001
						Online:  0x00000001
						Allowed: 0x00000001
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000001
							Online:  0x00000001
							Allowed: 0x00000001
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000001
								Online:  0x00000001
								Allowed: 0x00000001
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x00000055
			Online:  0x00000055
			Allowed: 0x00000055
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x00000055
				Online:  0x00000055
				Allowed: 0x00000055
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000001
					Online:  0x00000001
					Allowed: 0x00000001
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000001
						Online:  0x00000001
						Allowed: 0x00000001
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000001
							Online:  0x00000001
							Allowed: 0x00000001
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000001
								Online:  0x00000001
								Allowed: 0x00000001
[cn2:01042] mca:oob:select: Found 1 active transports
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000004
					Online:  0x00000004
					Allowed: 0x00000004
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000004
						Online:  0x00000004
						Allowed: 0x00000004
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000004
							Online:  0x00000004
							Allowed: 0x00000004
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000004
								Online:  0x00000004
								Allowed: 0x00000004
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000010
					Online:  0x00000010
					Allowed: 0x00000010
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000010
						Online:  0x00000010
						Allowed: 0x00000010
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000004
					Online:  0x00000004
					Allowed: 0x00000004
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000004
						Online:  0x00000004
						Allowed: 0x00000004
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000004
							Online:  0x00000004
							Allowed: 0x00000004
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000004
								Online:  0x00000004
								Allowed: 0x00000004
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000010
					Online:  0x00000010
					Allowed: 0x00000010
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000010
						Online:  0x00000010
						Allowed: 0x00000010
[cn2:01042] mca: base: components_register: registering rml components
[cn2:01042] mca: base: components_register: found loaded component oob
[cn2:01042] mca: base: components_register: component oob has no register or open function
[cn2:01042] mca: base: components_open: opening rml components
[cn2:01042] mca: base: components_open: found loaded component oob
[cn2:01042] mca: base: components_open: component oob open function successful
[cn2:01042] orte_rml_base_select: initializing rml component oob
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000010
							Online:  0x00000010
							Allowed: 0x00000010
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000010
								Online:  0x00000010
								Allowed: 0x00000010
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000040
					Online:  0x00000040
					Allowed: 0x00000040
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000040
						Online:  0x00000040
						Allowed: 0x00000040
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000040
							Online:  0x00000040
							Allowed: 0x00000040
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000040
								Online:  0x00000040
								Allowed: 0x00000040
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12582912KB
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000010
							Online:  0x00000010
							Allowed: 0x00000010
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000010
								Online:  0x00000010
								Allowed: 0x00000010
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000040
					Online:  0x00000040
					Allowed: 0x00000040
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000040
						Online:  0x00000040
						Allowed: 0x00000040
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000040
							Online:  0x00000040
							Allowed: 0x00000040
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000040
								Online:  0x00000040
								Allowed: 0x00000040
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12582912KB
		total=12582912KB
		total=12582912KB
		Cpuset:  0x000000aa
		Online:  0x000000aa
		Allowed: 0x000000aa
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x000000aa
			Online:  0x000000aa
			Allowed: 0x000000aa
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x000000aa
				Online:  0x000000aa
				Allowed: 0x000000aa
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000002
					Online:  0x00000002
					Allowed: 0x00000002
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000002
						Online:  0x00000002
						Allowed: 0x00000002
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000002
							Online:  0x00000002
							Allowed: 0x00000002
							Type: PU Number of child objects: 0
								Name=NULL
		Cpuset:  0x000000aa
		Online:  0x000000aa
		Allowed: 0x000000aa
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x000000aa
			Online:  0x000000aa
			Allowed: 0x000000aa
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x000000aa
				Online:  0x000000aa
				Allowed: 0x000000aa
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000002
					Online:  0x00000002
					Allowed: 0x00000002
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000002
						Online:  0x00000002
						Allowed: 0x00000002
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000002
							Online:  0x00000002
							Allowed: 0x00000002
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000002
								Online:  0x00000002
								Allowed: 0x00000002
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000008
					Online:  0x00000008
					Allowed: 0x00000008
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000008
						Online:  0x00000008
						Allowed: 0x00000008
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000008
							Online:  0x00000008
							Allowed: 0x00000008
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000008
								Online:  0x00000008
								Allowed: 0x00000008
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000020
					Online:  0x00000020
					Allowed: 0x00000020
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
								Cpuset:  0x00000002
								Online:  0x00000002
								Allowed: 0x00000002
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000008
					Online:  0x00000008
					Allowed: 0x00000008
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000008
						Online:  0x00000008
						Allowed: 0x00000008
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000008
							Online:  0x00000008
							Allowed: 0x00000008
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000008
								Online:  0x00000008
								Allowed: 0x00000008
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000020
					Online:  0x00000020
					Allowed: 0x00000020
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000020
						Online:  0x00000020
						Allowed: 0x00000020
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000020
							Online:  0x00000020
							Allowed: 0x00000020
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000020
								Online:  0x00000020
								Allowed: 0x00000020
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000080
					Online:  0x00000080
					Allowed: 0x00000080
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000080
						Online:  0x00000080
						Allowed: 0x00000080
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000080
							Online:  0x00000080
							Allowed: 0x00000080
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000080
								Online:  0x00000080
								Allowed: 0x00000080
						ways=8
						Cpuset:  0x00000020
						Online:  0x00000020
						Allowed: 0x00000020
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000020
							Online:  0x00000020
							Allowed: 0x00000020
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000020
								Online:  0x00000020
								Allowed: 0x00000020
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000080
					Online:  0x00000080
					Allowed: 0x00000080
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000080
						Online:  0x00000080
						Allowed: 0x00000080
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000080
							Online:  0x00000080
							Allowed: 0x00000080
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000080
								Online:  0x00000080
								Allowed: 0x00000080
[cn2:01042] mca: base: components_register: registering routed components
[cn2:01042] mca: base: components_register: found loaded component binomial
[cn2:01042] mca: base: components_register: component binomial has no register or open function
[cn2:01042] mca: base: components_register: found loaded component debruijn
[cn2:01042] mca: base: components_register: component debruijn has no register or open function
[cn2:01042] mca: base: components_register: found loaded component direct
[cn2:01042] mca: base: components_register: component direct has no register or open function
[cn2:01042] mca: base: components_register: found loaded component radix
[cn2:01042] mca: base: components_register: component radix register function successful
[cn2:01042] mca: base: components_open: opening routed components
[cn2:01042] mca: base: components_open: found loaded component binomial
[cn2:01042] mca: base: components_open: found loaded component debruijn
[cn2:01042] mca: base: components_open: found loaded component direct
[cn2:01042] mca: base: components_open: found loaded component radix
[cn2:01042] mca:base:select: Auto-selecting routed components
[cn2:01042] mca:base:select:(routed) Querying component [binomial]
[cn2:01042] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn2:01042] mca:base:select:(routed) Querying component [debruijn]
[cn2:01042] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn2:01042] mca:base:select:(routed) Querying component [direct]
[cn2:01042] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn2:01042] mca:base:select:(routed) Querying component [radix]
[cn2:01042] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn2:01042] mca:base:select:(routed) Selected component [binomial]
[cn2:01042] mca: base: close: unloading component debruijn
[cn2:01042] mca: base: close: unloading component direct
[cn2:01042] mca: base: close: unloading component radix
[cn2:01042] orte_routed_base_select: initializing selected component binomial
[cn5:14171] mca: base: components_register: registering oob components
[cn5:14171] mca: base: components_register: found loaded component tcp
[cn5:14171] mca: base: components_register: component tcp register function successful
[cn5:14171] mca: base: components_open: opening oob components
[cn5:14171] mca: base: components_open: found loaded component tcp
[cn5:14171] mca: base: components_open: component tcp open function successful
[cn5:14171] mca:oob:select: checking available component tcp
[cn5:14171] mca:oob:select: Querying component [tcp]
[cn5:14171] oob:tcp: component_available called
[cn5:14171] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn5:14171] [[15336,0],4] oob:tcp:init rejecting loopback interface lo
[cn5:14171] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn5:14171] [[15336,0],4] oob:tcp:init creating module for V4 address on interface eth0
[cn5:14171] [[15336,0],4] creating OOB-TCP module for interface eth0
[cn5:14171] [[15336,0],4] oob:tcp:init adding 192.168.1.5 to our list of V4 connections
[cn5:14171] [[15336,0],4] TCP STARTUP
[cn5:14171] [[15336,0],4] attempting to bind to IPv4 port 0
[cn5:14171] [[15336,0],4] assigned IPv4 port 34732
[cn5:14171] mca:oob:select: Adding component to end
[cn5:14171] mca:oob:select: Found 1 active transports
[cn5:14171] mca: base: components_register: registering rml components
[cn5:14171] mca: base: components_register: found loaded component oob
[cn5:14171] mca: base: components_register: component oob has no register or open function
[cn5:14171] mca: base: components_open: opening rml components
[cn5:14171] mca: base: components_open: found loaded component oob
[cn5:14171] mca: base: components_open: component oob open function successful
[cn5:14171] orte_rml_base_select: initializing rml component oob
[cn2:01042] mca: base: components_register: registering dstore components
[cn2:01042] mca: base: components_register: found loaded component hash
[cn2:01042] mca: base: components_register: component hash register function successful
[cn2:01042] mca: base: components_open: opening dstore components
[cn2:01042] mca: base: components_open: found loaded component hash
[cn2:01042] mca:dstore:select: checking available component hash
[cn2:01042] mca:dstore:select: Querying component [hash]
[cn2:01042] mca: base: components_register: registering grpcomm components
[cn2:01042] mca: base: components_register: found loaded component bad
[cn2:01042] mca: base: components_register: component bad has no register or open function
[cn2:01042] mca: base: components_register: found loaded component pmi
[cn2:01042] mca: base: components_register: component pmi register function successful
[cn2:01042] mca: base: components_open: opening grpcomm components
[cn2:01042] mca: base: components_open: found loaded component bad
[cn2:01042] mca: base: components_open: component bad open function successful
[cn2:01042] mca: base: components_open: found loaded component pmi
[cn2:01042] mca: base: components_open: component pmi open function successful
[cn2:01042] mca:base:select: Auto-selecting grpcomm components
[cn2:01042] mca:base:select:(grpcomm) Querying component [bad]
[cn2:01042] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn2:01042] mca:base:select:(grpcomm) Querying component [pmi]
[cn2:01042] mca:base:select:(grpcomm) Skipping component [pmi]. Query failed to return a module
[cn2:01042] mca:base:select:(grpcomm) Selected component [bad]
[cn2:01042] mca: base: close: component pmi closed
[cn2:01042] mca: base: close: unloading component pmi
[cn2:01042] [[15336,0],1] grpcomm:base:receive start comm
[cn2:01042] [[15336,0],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn2:01042] [[15336,0],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 15
[cn2:01042] [[15336,0],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 32
[cn2:01042] [[15336,0],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 11
[cn5:14171] mca: base: components_register: registering routed components
[cn5:14171] mca: base: components_register: found loaded component binomial
[cn5:14171] mca: base: components_register: component binomial has no register or open function
[cn5:14171] mca: base: components_register: found loaded component debruijn
[cn5:14171] mca: base: components_register: component debruijn has no register or open function
[cn5:14171] mca: base: components_register: found loaded component direct
[cn5:14171] mca: base: components_register: component direct has no register or open function
[cn5:14171] mca: base: components_register: found loaded component radix
[cn5:14171] mca: base: components_register: component radix register function successful
[cn5:14171] mca: base: components_open: opening routed components
[cn5:14171] mca: base: components_open: found loaded component binomial
[cn5:14171] mca: base: components_open: found loaded component debruijn
[cn5:14171] mca: base: components_open: found loaded component direct
[cn5:14171] mca: base: components_open: found loaded component radix
[cn5:14171] mca:base:select: Auto-selecting routed components
[cn5:14171] mca:base:select:(routed) Querying component [binomial]
[cn5:14171] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn5:14171] mca:base:select:(routed) Querying component [debruijn]
[cn5:14171] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn5:14171] mca:base:select:(routed) Querying component [direct]
[cn5:14171] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn5:14171] mca:base:select:(routed) Querying component [radix]
[cn5:14171] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn5:14171] mca:base:select:(routed) Selected component [binomial]
[cn5:14171] mca: base: close: unloading component debruijn
[cn5:14171] mca: base: close: unloading component direct
[cn5:14171] mca: base: close: unloading component radix
[cn5:14171] orte_routed_base_select: initializing selected component binomial
[cn2:01042] [[15336,0],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn2:01042] [[15336,0],1] routed:binomial rank 0 parent 0 me 1 num_procs 5
[cn2:01042] [[15336,0],1] routed:binomial find children of rank 0
[cn2:01042] [[15336,0],1] routed:binomial find children checking peer 1
[cn2:01042] [[15336,0],1] routed:binomial find children computing tree
[cn2:01042] [[15336,0],1] routed:binomial rank 1 parent 0 me 1 num_procs 5
[cn2:01042] [[15336,0],1] routed:binomial 1 found child 3
[cn2:01042] [[15336,0],1] routed:binomial rank 0 parent 0 me 3 num_procs 5
[cn2:01042] [[15336,0],1] routed:binomial find children of rank 0
[cn2:01042] [[15336,0],1] routed:binomial find children checking peer 1
[cn2:01042] [[15336,0],1] routed:binomial find children computing tree
[cn2:01042] [[15336,0],1] routed:binomial rank 1 parent 0 me 3 num_procs 5
[cn2:01042] [[15336,0],1] routed:binomial find children of rank 1
[cn2:01042] [[15336,0],1] routed:binomial find children checking peer 3
[cn2:01042] [[15336,0],1] routed:binomial find children computing tree
[cn2:01042] [[15336,0],1] routed:binomial rank 3 parent 1 me 3 num_procs 5
[cn2:01042] [[15336,0],1] routed:binomial find children returning found value 1
[cn2:01042] [[15336,0],1] routed:binomial find children returning found value 1
[cn2:01042] [[15336,0],1] routed:binomial find children returning found value 0
[cn2:01042] [[15336,0],1]: parent 0 num_children 1
[cn2:01042] [[15336,0],1]: 	child 3
[cn2:01042] [[15336,0],1] setting up session dir with
	tmpdir: UNDEF
	host cn2
[cn5:14171] mca: base: components_register: registering dstore components
[cn5:14171] mca: base: components_register: found loaded component hash
[cn5:14171] mca: base: components_register: component hash register function successful
[cn5:14171] mca: base: components_open: opening dstore components
[cn5:14171] mca: base: components_open: found loaded component hash
[cn5:14171] mca:dstore:select: checking available component hash
[cn5:14171] mca:dstore:select: Querying component [hash]
[cn5:14171] mca: base: components_register: registering grpcomm components
[cn5:14171] mca: base: components_register: found loaded component bad
[cn5:14171] mca: base: components_register: component bad has no register or open function
[cn5:14171] mca: base: components_register: found loaded component pmi
[cn5:14171] mca: base: components_register: component pmi register function successful
[cn5:14171] mca: base: components_open: opening grpcomm components
[cn5:14171] mca: base: components_open: found loaded component bad
[cn5:14171] mca: base: components_open: component bad open function successful
[cn5:14171] mca: base: components_open: found loaded component pmi
[cn5:14171] mca: base: components_open: component pmi open function successful
[cn5:14171] mca:base:select: Auto-selecting grpcomm components
[cn5:14171] mca:base:select:(grpcomm) Querying component [bad]
[cn5:14171] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn5:14171] mca:base:select:(grpcomm) Querying component [pmi]
[cn5:14171] mca:base:select:(grpcomm) Skipping component [pmi]. Query failed to return a module
[cn5:14171] mca:base:select:(grpcomm) Selected component [bad]
[cn5:14171] mca: base: close: component pmi closed
[cn5:14171] mca: base: close: unloading component pmi
[cn5:14171] [[15336,0],4] grpcomm:base:receive start comm
[cn5:14171] [[15336,0],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn5:14171] [[15336,0],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 15
[cn5:14171] [[15336,0],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 32
[cn5:14171] [[15336,0],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 11
[cn5:14171] [[15336,0],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn5:14171] [[15336,0],4] routed:binomial rank 0 parent 0 me 4 num_procs 5
[cn5:14171] [[15336,0],4] routed:binomial find children of rank 0
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 1
[cn5:14171] [[15336,0],4] routed:binomial find children computing tree
[cn5:14171] [[15336,0],4] routed:binomial rank 1 parent 0 me 4 num_procs 5
[cn5:14171] [[15336,0],4] routed:binomial find children of rank 1
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 3
[cn5:14171] [[15336,0],4] routed:binomial find children computing tree
[cn5:14171] [[15336,0],4] routed:binomial rank 3 parent 1 me 4 num_procs 5
[cn5:14171] [[15336,0],4] routed:binomial find children of rank 3
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 7
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 5
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 2
[cn5:14171] [[15336,0],4] routed:binomial find children computing tree
[cn5:14171] [[15336,0],4] routed:binomial rank 2 parent 0 me 4 num_procs 5
[cn5:14171] [[15336,0],4] routed:binomial find children of rank 2
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 6
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 4
[cn5:14171] [[15336,0],4] routed:binomial find children computing tree
[cn5:14171] [[15336,0],4] routed:binomial rank 4 parent 0 me 4 num_procs 5
[cn5:14171] [[15336,0],4] routed:binomial find children returning found value 0
[cn5:14171] [[15336,0],4]: parent 0 num_children 0
[cn5:14171] [[15336,0],4] setting up session dir with
	tmpdir: UNDEF
	host cn5
[cn4:17828] mca: base: components_register: registering oob components
[cn4:17828] mca: base: components_register: found loaded component tcp
[cn4:17828] mca: base: components_register: component tcp register function successful
[cn4:17828] mca: base: components_open: opening oob components
[cn4:17828] mca: base: components_open: found loaded component tcp
[cn4:17828] mca: base: components_open: component tcp open function successful
[cn4:17828] mca:oob:select: checking available component tcp
[cn4:17828] mca:oob:select: Querying component [tcp]
[cn4:17828] oob:tcp: component_available called
[cn4:17828] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn4:17828] [[15336,0],3] oob:tcp:init rejecting loopback interface lo
[cn4:17828] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn4:17828] [[15336,0],3] oob:tcp:init creating module for V4 address on interface eth0
[cn4:17828] [[15336,0],3] creating OOB-TCP module for interface eth0
[cn4:17828] [[15336,0],3] oob:tcp:init adding 192.168.1.4 to our list of V4 connections
[cn4:17828] [[15336,0],3] TCP STARTUP
[cn4:17828] [[15336,0],3] attempting to bind to IPv4 port 0
[cn4:17828] [[15336,0],3] assigned IPv4 port 36268
[cn4:17828] mca:oob:select: Adding component to end
[cn4:17828] mca:oob:select: Found 1 active transports
[cn4:17828] mca: base: components_register: registering rml components
[cn4:17828] mca: base: components_register: found loaded component oob
[cn4:17828] mca: base: components_register: component oob has no register or open function
[cn4:17828] mca: base: components_open: opening rml components
[cn4:17828] mca: base: components_open: found loaded component oob
[cn4:17828] mca: base: components_open: component oob open function successful
[cn4:17828] orte_rml_base_select: initializing rml component oob
[cn3:21416] mca: base: components_register: registering oob components
[cn3:21416] mca: base: components_register: found loaded component tcp
[cn3:21416] mca: base: components_register: component tcp register function successful
[cn3:21416] mca: base: components_open: opening oob components
[cn3:21416] mca: base: components_open: found loaded component tcp
[cn3:21416] mca: base: components_open: component tcp open function successful
[cn3:21416] mca:oob:select: checking available component tcp
[cn3:21416] mca:oob:select: Querying component [tcp]
[cn3:21416] oob:tcp: component_available called
[cn3:21416] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn3:21416] [[15336,0],2] oob:tcp:init rejecting loopback interface lo
[cn3:21416] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn3:21416] [[15336,0],2] oob:tcp:init creating module for V4 address on interface eth0
[cn3:21416] [[15336,0],2] creating OOB-TCP module for interface eth0
[cn3:21416] [[15336,0],2] oob:tcp:init adding 192.168.1.3 to our list of V4 connections
[cn3:21416] [[15336,0],2] TCP STARTUP
[cn3:21416] [[15336,0],2] attempting to bind to IPv4 port 0
[cn3:21416] [[15336,0],2] assigned IPv4 port 42008
[cn3:21416] mca:oob:select: Adding component to end
[cn3:21416] mca:oob:select: Found 1 active transports
[cn3:21416] mca: base: components_register: registering rml components
[cn3:21416] mca: base: components_register: found loaded component oob
[cn3:21416] mca: base: components_register: component oob has no register or open function
[cn3:21416] mca: base: components_open: opening rml components
[cn3:21416] mca: base: components_open: found loaded component oob
[cn3:21416] mca: base: components_open: component oob open function successful
[cn3:21416] orte_rml_base_select: initializing rml component oob
[cn2:01042] procdir: /tmp/openmpi-sessions-artpol@cn2_0/15336/0/1
[cn2:01042] jobdir: /tmp/openmpi-sessions-artpol@cn2_0/15336/0
[cn2:01042] top: openmpi-sessions-artpol@cn2_0
[cn2:01042] tmp: /tmp
[cn2:01042] sess_dir_cleanup: job session dir does not exist
[cn2:01042] procdir: /tmp/openmpi-sessions-artpol@cn2_0/15336/0/1
[cn2:01042] jobdir: /tmp/openmpi-sessions-artpol@cn2_0/15336/0
[cn2:01042] top: openmpi-sessions-artpol@cn2_0
[cn2:01042] tmp: /tmp
[cn2:01042] [[15336,0],1] routed_binomial: init routes for daemon job [15336,0]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn2:01042] [[15336,0],1] routed_binomial: completed init routes
[cn2:01042] [[15336,0],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 3
[cn2:01042] [[15336,0],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 21
[cn2:01042] [[15336,0],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 45
[cn2:01042] [[15336,0],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 46
[cn2:01042] [[15336,0],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 1
Daemon [[15336,0],1] checking in as pid 1042 on host cn2
[cn2:01042] progress: progress_set_yield_when_idle to false
[cn2:01042] progress: set_event_flag setting to 1
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 10
[cn2:01042] [[15336,0],1] orted: up and running - waiting for commands!
[cn2:01042] [[15336,0],1] posting recv
[cn2:01042] [[15336,0],1] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn2:01042] [[15336,0],1] posting recv
[cn2:01042] [[15336,0],1] posting persistent recv on tag 15 for peer [[WILDCARD],WILDCARD]
[cn2:01042] [[15336,0],1] posting recv
[cn2:01042] [[15336,0],1] posting persistent recv on tag 32 for peer [[WILDCARD],WILDCARD]
[cn2:01042] [[15336,0],1] posting recv
[cn2:01042] [[15336,0],1] posting persistent recv on tag 11 for peer [[WILDCARD],WILDCARD]
[cn2:01042] [[15336,0],1] posting recv
[cn2:01042] [[15336,0],1] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn2:01042] [[15336,0],1]: set_addr to uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn2:01042] [[15336,0],1]:set_addr checking if peer [[15336,0],0] is reachable via component tcp
[cn2:01042] [[15336,0],1] oob:tcp: working peer [[15336,0],0] address tcp://192.168.1.1,192.168.10.2:37347
[cn2:01042] [[15336,0],1] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn2:01042] [[15336,0],1] PASSING ADDR 192.168.1.1 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn2:01042] [[15336,0],1]:tcp set addr for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] UNFOUND KERNEL INDEX -13 FOR ADDRESS 192.168.10.2
[cn2:01042] [[15336,0],1]: peer [[15336,0],0] is reachable via component tcp
[cn2:01042] [[15336,0],1] posting recv
[cn2:01042] [[15336,0],1] posting persistent recv on tag 3 for peer [[WILDCARD],WILDCARD]
[cn2:01042] [[15336,0],1] posting recv
[cn2:01042] [[15336,0],1] posting persistent recv on tag 21 for peer [[WILDCARD],WILDCARD]
[cn2:01042] [[15336,0],1] posting recv
[cn2:01042] [[15336,0],1] posting persistent recv on tag 45 for peer [[WILDCARD],WILDCARD]
[cn2:01042] [[15336,0],1] posting recv
[cn2:01042] [[15336,0],1] posting persistent recv on tag 46 for peer [[WILDCARD],WILDCARD]
[cn2:01042] [[15336,0],1] posting recv
[cn2:01042] [[15336,0],1] posting persistent recv on tag 1 for peer [[WILDCARD],WILDCARD]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 10
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:tcp:processing set_peer cmd for interface eth0
[cn2:01042] [[15336,0],1] SET_PEER ADDING PEER [[15336,0],0]
[cn2:01042] [[15336,0],1] set_peer: peer [[15336,0],0] is listening on interface eth0 net 192.168.1.1 port 37347
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:10
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:10
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: initiating connection to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],0]
[cn2:01042] [[15336,0],1] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0
[cn1:14209] [[15336,0],0] mca_oob_tcp_listen_thread: new connection: (14, 0) 192.168.1.2:43649
[cn1:14209] [[15336,0],0] connection_handler: working connection (14, 2) 192.168.1.2:43649
[cn1:14209] [[15336,0],0] accept_connection: 192.168.1.2:43649
[cn1:14209] [[15336,0],0]:tcp:recv:handler called
[cn1:14209] [[15336,0],0] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 14
[cn1:14209] [[15336,0],0] waiting for connect ack from UNKNOWN
[cn1:14209] [[15336,0],0] connect ack received from UNKNOWN
[cn1:14209] [[15336,0],0] connect-ack recvd from UNKNOWN
[cn1:14209] [[15336,0],0] mca_oob_tcp_recv_connect: connection from new peer
[cn1:14209] [[15336,0],0] connect-ack header from [[15336,0],1] is okay
[cn1:14209] [[15336,0],0] waiting for connect ack from [[15336,0],1]
[cn1:14209] [[15336,0],0] connect ack received from [[15336,0],1]
[cn1:14209] [[15336,0],0] connect-ack version from [[15336,0],1] matches ours
[cn1:14209] [[15336,0],0] connect-ack [[15336,0],1] authenticated
[cn1:14209] [[15336,0],0] tcp:peer_accept called for peer [[15336,0],1] in state UNKNOWN on socket 14
[cn1:14209] [[15336,0],0] SEND CONNECT ACK
[cn1:14209] fetching data from INTERNAL dstore
[cn1:14209] dstore:hash:fetch: searching for key opal.cred on proc 1005060096
[cn1:14209] dstore_hash:fetch key opal.cred for proc 1005060096 not found
[cn1:14209] [[15336,0],0] send blocking of 47 bytes to socket 14
[cn1:14209] [[15336,0],0] blocking send complete to socket 14
[cn1:14209] [[15336,0],0]-[[15336,0],1] tcp_peer_connected on socket 14
[cn1:14209] [[15336,0],0] routed_binomial_update: [[15336,0],1] --> [[15336,0],1]
[cn1:14209] [[15336,0],0]-[[15336,0],1] accepted: 192.168.1.1 - 192.168.1.2 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn1:14209] [[15336,0],0] tcp:set_module called for peer [[15336,0],1]
[cn4:17828] mca: base: components_register: registering routed components
[cn4:17828] mca: base: components_register: found loaded component binomial
[cn4:17828] mca: base: components_register: component binomial has no register or open function
[cn4:17828] mca: base: components_register: found loaded component debruijn
[cn4:17828] mca: base: components_register: component debruijn has no register or open function
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 13098
[cn4:17828] mca: base: components_register: found loaded component direct
[cn4:17828] mca: base: components_register: component direct has no register or open function
[cn4:17828] mca: base: components_register: found loaded component radix
[cn2:01042] [[15336,0],1] oob:tcp:peer creating socket to [[15336,0],0]
[cn4:17828] mca: base: components_register: component radix register function successful
[cn2:01042] [[15336,0],1] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0 on socket 9
[cn4:17828] mca: base: components_open: opening routed components
[cn4:17828] mca: base: components_open: found loaded component binomial
[cn4:17828] mca: base: components_open: found loaded component debruijn
[cn4:17828] mca: base: components_open: found loaded component direct
[cn4:17828] mca: base: components_open: found loaded component radix
[cn4:17828] mca:base:select: Auto-selecting routed components
[cn4:17828] mca:base:select:(routed) Querying component [binomial]
[cn2:01042] [[15336,0],1] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] on 192.168.1.1:37347 - 0 retries
[cn2:01042] [[15336,0],1] waiting for connect completion to [[15336,0],0] - activating send event
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler CONNECTING
[cn2:01042] [[15336,0],1]:tcp:complete_connect called for peer [[15336,0],0] on socket 9
[cn2:01042] [[15336,0],1] tcp_peer_complete_connect: sending ack to [[15336,0],0]
[cn2:01042] [[15336,0],1] SEND CONNECT ACK
[cn2:01042] fetching data from INTERNAL dstore
[cn2:01042] dstore:hash:fetch: searching for key opal.cred on proc 5300027392
[cn2:01042] dstore_hash:fetch key opal.cred for proc 5300027392 not found
[cn2:01042] [[15336,0],1] send blocking of 47 bytes to socket 9
[cn2:01042] [[15336,0],1] blocking send complete to socket 9
[cn4:17828] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn4:17828] mca:base:select:(routed) Querying component [debruijn]
[cn4:17828] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn4:17828] mca:base:select:(routed) Querying component [direct]
[cn4:17828] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn4:17828] mca:base:select:(routed) Querying component [radix]
[cn4:17828] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn4:17828] mca:base:select:(routed) Selected component [binomial]
[cn4:17828] mca: base: close: unloading component debruijn
[cn2:01042] [[15336,0],1] tcp_peer_complete_connect: setting read event on connection to [[15336,0],0]
[cn4:17828] mca: base: close: unloading component direct
[cn4:17828] mca: base: close: unloading component radix
[cn4:17828] orte_routed_base_select: initializing selected component binomial
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 13098 BYTES FOR DEST [[15336,0],0] TAG 10
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 13098 bytes from [[15336,0],1] for tag 10
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch from daemon [[15336,0],1]
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch from daemon [[15336,0],1] on node cn2
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch attempting to assign daemon [[15336,0],1] to node cn2
[cn1:14209] [[15336,0],0] RECEIVED TOPOLOGY FROM NODE cn2
[cn1:14209] Type: Machine Number of child objects: 2
	Name=NULL
	total=25100516KB
	DMIProductName=S5520UR
	DMIProductVersion=....................
	DMIBoardVendor="Intel Corporation"
	DMIBoardName=S5520UR
	DMIBoardVersion=E22554-751
	DMIBoardAssetTag=....................
	DMIChassisVendor=...............................
	DMIChassisType=23
	DMIChassisVersion=..................
	DMIChassisAssetTag=....................
	DMIBIOSVendor="Intel Corp."
	DMIBIOSVersion=S5500.86B.01.00.0054.092820101104
	DMIBIOSDate=09/28/2010
	DMISysVendor="Intel Corporation"
	Backend=Linux
	LinuxCgroup=/
	OSName=Linux
	OSRelease=2.6.32-431.3.1.el6.x86_64
	OSVersion="#1 SMP Fri Jan 3 21:39:27 UTC 2014"
	Architecture=x86_64
	Cpuset:  0x000000ff
	Online:  0x000000ff
	Allowed: 0x000000ff
	Bind CPU proc:   TRUE
	Bind CPU thread: TRUE
	Bind MEM proc:   FALSE
	Bind MEM thread: FALSE
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12517604KB
		total=12517604KB
		Cpuset:  0x00000055
		Online:  0x00000055
		Allowed: 0x00000055
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x00000055
			Online:  0x00000055
			Allowed: 0x00000055
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x00000055
				Online:  0x00000055
				Allowed: 0x00000055
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000001
					Online:  0x00000001
					Allowed: 0x00000001
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000001
						Online:  0x00000001
						Allowed: 0x00000001
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000001
							Online:  0x00000001
							Allowed: 0x00000001
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000001
								Online:  0x00000001
								Allowed: 0x00000001
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000004
					Online:  0x00000004
					Allowed: 0x00000004
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000004
						Online:  0x00000004
						Allowed: 0x00000004
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000004
							Online:  0x00000004
							Allowed: 0x00000004
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000004
								Online:  0x00000004
								Allowed: 0x00000004
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000010
					Online:  0x00000010
					Allowed: 0x00000010
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000010
						Online:  0x00000010
						Allowed: 0x00000010
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000010
							Online:  0x00000010
							Allowed: 0x00000010
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000010
								Online:  0x00000010
								Allowed: 0x00000010
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000040
					Online:  0x00000040
					Allowed: 0x00000040
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000040
						Online:  0x00000040
						Allowed: 0x00000040
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000040
							Online:  0x00000040
							Allowed: 0x00000040
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000040
								Online:  0x00000040
								Allowed: 0x00000040
	Type: NUMANode Number of child objects: 1
		Name=NULL
		local=12582912KB
		total=12582912KB
		Cpuset:  0x000000aa
		Online:  0x000000aa
		Allowed: 0x000000aa
		Type: Socket Number of child objects: 1
			Name=NULL
			CPUModel="Intel(R) Xeon(R) CPU           E5620  @ 2.40GHz"
			Cpuset:  0x000000aa
			Online:  0x000000aa
			Allowed: 0x000000aa
			Type: L3Cache Number of child objects: 4
				Name=NULL
				size=12288KB
				linesize=64
				ways=16
				Cpuset:  0x000000aa
				Online:  0x000000aa
				Allowed: 0x000000aa
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000002
					Online:  0x00000002
					Allowed: 0x00000002
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000002
						Online:  0x00000002
						Allowed: 0x00000002
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000002
							Online:  0x00000002
							Allowed: 0x00000002
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000002
								Online:  0x00000002
								Allowed: 0x00000002
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000008
					Online:  0x00000008
					Allowed: 0x00000008
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000008
						Online:  0x00000008
						Allowed: 0x00000008
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000008
							Online:  0x00000008
							Allowed: 0x00000008
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000008
								Online:  0x00000008
								Allowed: 0x00000008
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000020
					Online:  0x00000020
					Allowed: 0x00000020
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000020
						Online:  0x00000020
						Allowed: 0x00000020
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000020
							Online:  0x00000020
							Allowed: 0x00000020
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000020
								Online:  0x00000020
								Allowed: 0x00000020
				Type: L2Cache Number of child objects: 1
					Name=NULL
					size=256KB
					linesize=64
					ways=8
					Cpuset:  0x00000080
					Online:  0x00000080
					Allowed: 0x00000080
					Type: L1dCache Number of child objects: 1
						Name=NULL
						size=32KB
						linesize=64
						ways=8
						Cpuset:  0x00000080
						Online:  0x00000080
						Allowed: 0x00000080
						Type: Core Number of child objects: 1
							Name=NULL
							Cpuset:  0x00000080
							Online:  0x00000080
							Allowed: 0x00000080
							Type: PU Number of child objects: 0
								Name=NULL
								Cpuset:  0x00000080
								Online:  0x00000080
								Allowed: 0x00000080
[cn1:14209] [[15336,0],0] mca_oob_tcp_listen_thread: new connection: (15, 11) 192.168.1.5:40124
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] RECV CONNECT ACK FROM [[15336,0],0] ON SOCKET 9
[cn2:01042] [[15336,0],1] waiting for connect ack from [[15336,0],0]
[cn2:01042] [[15336,0],1] connect ack received from [[15336,0],0]
[cn2:01042] [[15336,0],1] connect-ack recvd from [[15336,0],0]
[cn2:01042] [[15336,0],1] connect-ack header from [[15336,0],0] is okay
[cn2:01042] [[15336,0],1] waiting for connect ack from [[15336,0],0]
[cn2:01042] [[15336,0],1] connect ack received from [[15336,0],0]
[cn2:01042] [[15336,0],1] connect-ack version from [[15336,0],0] matches ours
[cn2:01042] [[15336,0],1] connect-ack [[15336,0],0] authenticated
[cn2:01042] [[15336,0],1]-[[15336,0],0] tcp_peer_connected on socket 9
[cn2:01042] [[15336,0],1] routed_binomial_update: [[15336,0],0] --> [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] connected: 192.168.1.2 - 192.168.1.1 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn5:14171] procdir: /tmp/openmpi-sessions-artpol@cn5_0/15336/0/4
[cn5:14171] jobdir: /tmp/openmpi-sessions-artpol@cn5_0/15336/0
[cn5:14171] top: openmpi-sessions-artpol@cn5_0
[cn5:14171] tmp: /tmp
[cn3:21416] mca: base: components_register: registering routed components
[cn3:21416] mca: base: components_register: found loaded component binomial
[cn3:21416] mca: base: components_register: component binomial has no register or open function
[cn3:21416] mca: base: components_register: found loaded component debruijn
[cn3:21416] mca: base: components_register: component debruijn has no register or open function
[cn3:21416] mca: base: components_register: found loaded component direct
[cn3:21416] mca: base: components_register: component direct has no register or open function
[cn2:01042] [[15336,0],1]:tcp:recv:handler starting send/recv events
[cn2:01042] [[15336,0],1] tcp:set_module called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] sess_dir_cleanup: job session dir does not exist
[cn3:21416] mca: base: components_register: found loaded component radix
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 13098 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] procdir: /tmp/openmpi-sessions-artpol@cn5_0/15336/0/4
[cn5:14171] jobdir: /tmp/openmpi-sessions-artpol@cn5_0/15336/0
[cn5:14171] top: openmpi-sessions-artpol@cn5_0
[cn5:14171] tmp: /tmp
[cn5:14171] [[15336,0],4] routed_binomial: init routes for daemon job [15336,0]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn5:14171] [[15336,0],4] routed_binomial: completed init routes
[cn3:21416] mca: base: components_register: component radix register function successful
[cn5:14171] [[15336,0],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 3
[cn3:21416] mca: base: components_open: opening routed components
[cn3:21416] mca: base: components_open: found loaded component binomial
[cn5:14171] [[15336,0],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 21
[cn3:21416] mca: base: components_open: found loaded component debruijn
[cn3:21416] mca: base: components_open: found loaded component direct
[cn3:21416] mca: base: components_open: found loaded component radix
[cn3:21416] mca:base:select: Auto-selecting routed components
[cn3:21416] mca:base:select:(routed) Querying component [binomial]
[cn5:14171] [[15336,0],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 45
[cn5:14171] [[15336,0],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 46
[cn5:14171] [[15336,0],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 1
Daemon [[15336,0],4] checking in as pid 14171 on host cn5
[cn5:14171] progress: progress_set_yield_when_idle to false
[cn5:14171] progress: set_event_flag setting to 1
[cn3:21416] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn3:21416] mca:base:select:(routed) Querying component [debruijn]
[cn3:21416] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn3:21416] mca:base:select:(routed) Querying component [direct]
[cn3:21416] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn3:21416] mca:base:select:(routed) Querying component [radix]
[cn3:21416] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn3:21416] mca:base:select:(routed) Selected component [binomial]
[cn3:21416] mca: base: close: unloading component debruijn
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 10
[cn5:14171] [[15336,0],4] orted: up and running - waiting for commands!
[cn3:21416] mca: base: close: unloading component direct
[cn5:14171] [[15336,0],4] posting recv
[cn5:14171] [[15336,0],4] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn3:21416] mca: base: close: unloading component radix
[cn5:14171] [[15336,0],4] posting recv
[cn5:14171] [[15336,0],4] posting persistent recv on tag 15 for peer [[WILDCARD],WILDCARD]
[cn5:14171] [[15336,0],4] posting recv
[cn3:21416] orte_routed_base_select: initializing selected component binomial
[cn5:14171] [[15336,0],4] posting persistent recv on tag 32 for peer [[WILDCARD],WILDCARD]
[cn5:14171] [[15336,0],4] posting recv
[cn5:14171] [[15336,0],4] posting persistent recv on tag 11 for peer [[WILDCARD],WILDCARD]
[cn5:14171] [[15336,0],4] posting recv
[cn5:14171] [[15336,0],4] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn5:14171] [[15336,0],4]: set_addr to uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn5:14171] [[15336,0],4]:set_addr checking if peer [[15336,0],0] is reachable via component tcp
[cn5:14171] [[15336,0],4] oob:tcp: working peer [[15336,0],0] address tcp://192.168.1.1,192.168.10.2:37347
[cn5:14171] [[15336,0],4] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn5:14171] [[15336,0],4] PASSING ADDR 192.168.1.1 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn5:14171] [[15336,0],4]:tcp set addr for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] UNFOUND KERNEL INDEX -13 FOR ADDRESS 192.168.10.2
[cn5:14171] [[15336,0],4]: peer [[15336,0],0] is reachable via component tcp
[cn5:14171] [[15336,0],4] posting recv
[cn5:14171] [[15336,0],4] posting persistent recv on tag 3 for peer [[WILDCARD],WILDCARD]
[cn5:14171] [[15336,0],4] posting recv
[cn5:14171] [[15336,0],4] posting persistent recv on tag 21 for peer [[WILDCARD],WILDCARD]
[cn5:14171] [[15336,0],4] posting recv
[cn5:14171] [[15336,0],4] posting persistent recv on tag 45 for peer [[WILDCARD],WILDCARD]
[cn5:14171] [[15336,0],4] posting recv
[cn5:14171] [[15336,0],4] posting persistent recv on tag 46 for peer [[WILDCARD],WILDCARD]
[cn5:14171] [[15336,0],4] posting recv
[cn5:14171] [[15336,0],4] posting persistent recv on tag 1 for peer [[WILDCARD],WILDCARD]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 10
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]:tcp:processing set_peer cmd for interface eth0
[cn5:14171] [[15336,0],4] SET_PEER ADDING PEER [[15336,0],0]
[cn5:14171] [[15336,0],4] set_peer: peer [[15336,0],0] is listening on interface eth0 net 192.168.1.1 port 37347
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:10
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:10
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: initiating connection to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],0]
[cn5:14171] [[15336,0],4] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0
[cn5:14171] [[15336,0],4] oob:tcp:peer creating socket to [[15336,0],0]
[cn5:14171] [[15336,0],4] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0 on socket 9
[cn5:14171] [[15336,0],4] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] on 192.168.1.1:37347 - 0 retries
[cn5:14171] [[15336,0],4] waiting for connect completion to [[15336,0],0] - activating send event
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler CONNECTING
[cn5:14171] [[15336,0],4]:tcp:complete_connect called for peer [[15336,0],0] on socket 9
[cn5:14171] [[15336,0],4] tcp_peer_complete_connect: sending ack to [[15336,0],0]
[cn5:14171] [[15336,0],4] SEND CONNECT ACK
[cn5:14171] fetching data from INTERNAL dstore
[cn5:14171] dstore:hash:fetch: searching for key opal.cred on proc 18184929280
[cn5:14171] dstore_hash:fetch key opal.cred for proc 18184929280 not found
[cn5:14171] [[15336,0],4] send blocking of 47 bytes to socket 9
[cn5:14171] [[15336,0],4] blocking send complete to socket 9
[cn5:14171] [[15336,0],4] tcp_peer_complete_connect: setting read event on connection to [[15336,0],0]
[cn1:14209] [[15336,0],0] TOPOLOGY MATCHES - DISCARDING
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch completed for daemon [[15336,0],1] at contact 1005060096.1;tcp://192.168.1.2:51705
[cn1:14209] [[15336,0],0]: set_addr to uri 1005060096.1;tcp://192.168.1.2:51705
[cn1:14209] [[15336,0],0]:set_addr checking if peer [[15336,0],1] is reachable via component tcp
[cn1:14209] [[15336,0],0] oob:tcp: working peer [[15336,0],1] address tcp://192.168.1.2:51705
[cn1:14209] [[15336,0],0] PEER [[15336,0],1] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn1:14209] [[15336,0],0] PASSING ADDR 192.168.1.2 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn1:14209] [[15336,0],0]:tcp set addr for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]: peer [[15336,0],1] is reachable via component tcp
[cn1:14209] [[15336,0],0] connection_handler: working connection (15, 0) 192.168.1.5:40124
[cn1:14209] [[15336,0],0] accept_connection: 192.168.1.5:40124
[cn1:14209] [[15336,0],0]:tcp:processing set_peer cmd for interface eth0
[cn1:14209] [[15336,0],0] set_peer: peer [[15336,0],1] is listening on interface eth0 net 192.168.1.2 port 51705
[cn1:14209] [[15336,0],0]:tcp:recv:handler called
[cn1:14209] [[15336,0],0] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 15
[cn1:14209] [[15336,0],0] waiting for connect ack from UNKNOWN
[cn1:14209] [[15336,0],0] connect ack received from UNKNOWN
[cn1:14209] [[15336,0],0] connect-ack recvd from UNKNOWN
[cn1:14209] [[15336,0],0] mca_oob_tcp_recv_connect: connection from new peer
[cn1:14209] [[15336,0],0] connect-ack header from [[15336,0],4] is okay
[cn1:14209] [[15336,0],0] waiting for connect ack from [[15336,0],4]
[cn1:14209] [[15336,0],0] connect ack received from [[15336,0],4]
[cn1:14209] [[15336,0],0] connect-ack version from [[15336,0],4] matches ours
[cn1:14209] [[15336,0],0] connect-ack [[15336,0],4] authenticated
[cn1:14209] [[15336,0],0] tcp:peer_accept called for peer [[15336,0],4] in state UNKNOWN on socket 15
[cn1:14209] [[15336,0],0] SEND CONNECT ACK
[cn1:14209] [[15336,0],0] send blocking of 47 bytes to socket 15
[cn1:14209] [[15336,0],0] blocking send complete to socket 15
[cn1:14209] [[15336,0],0]-[[15336,0],4] tcp_peer_connected on socket 15
[cn1:14209] [[15336,0],0] routed_binomial_update: [[15336,0],4] --> [[15336,0],4]
[cn1:14209] [[15336,0],0]-[[15336,0],4] accepted: 192.168.1.1 - 192.168.1.5 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn1:14209] [[15336,0],0] tcp:set_module called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 97
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] RECV CONNECT ACK FROM [[15336,0],0] ON SOCKET 9
[cn5:14171] [[15336,0],4] waiting for connect ack from [[15336,0],0]
[cn5:14171] [[15336,0],4] connect ack received from [[15336,0],0]
[cn5:14171] [[15336,0],4] connect-ack recvd from [[15336,0],0]
[cn5:14171] [[15336,0],4] connect-ack header from [[15336,0],0] is okay
[cn5:14171] [[15336,0],4] waiting for connect ack from [[15336,0],0]
[cn5:14171] [[15336,0],4] connect ack received from [[15336,0],0]
[cn5:14171] [[15336,0],4] connect-ack version from [[15336,0],0] matches ours
[cn5:14171] [[15336,0],4] connect-ack [[15336,0],0] authenticated
[cn5:14171] [[15336,0],4]-[[15336,0],0] tcp_peer_connected on socket 9
[cn5:14171] [[15336,0],4] routed_binomial_update: [[15336,0],0] --> [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] connected: 192.168.1.5 - 192.168.1.1 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn5:14171] [[15336,0],4]:tcp:recv:handler starting send/recv events
[cn5:14171] [[15336,0],4] tcp:set_module called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 97 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] mca: base: components_register: registering dstore components
[cn4:17828] mca: base: components_register: found loaded component hash
[cn4:17828] mca: base: components_register: component hash register function successful
[cn4:17828] mca: base: components_open: opening dstore components
[cn4:17828] mca: base: components_open: found loaded component hash
[cn4:17828] mca:dstore:select: checking available component hash
[cn4:17828] mca:dstore:select: Querying component [hash]
[cn4:17828] mca: base: components_register: registering grpcomm components
[cn4:17828] mca: base: components_register: found loaded component bad
[cn4:17828] mca: base: components_register: component bad has no register or open function
[cn4:17828] mca: base: components_register: found loaded component pmi
[cn4:17828] mca: base: components_register: component pmi register function successful
[cn4:17828] mca: base: components_open: opening grpcomm components
[cn4:17828] mca: base: components_open: found loaded component bad
[cn4:17828] mca: base: components_open: component bad open function successful
[cn4:17828] mca: base: components_open: found loaded component pmi
[cn4:17828] mca: base: components_open: component pmi open function successful
[cn4:17828] mca:base:select: Auto-selecting grpcomm components
[cn4:17828] mca:base:select:(grpcomm) Querying component [bad]
[cn4:17828] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn4:17828] mca:base:select:(grpcomm) Querying component [pmi]
[cn4:17828] mca:base:select:(grpcomm) Skipping component [pmi]. Query failed to return a module
[cn4:17828] mca:base:select:(grpcomm) Selected component [bad]
[cn4:17828] mca: base: close: component pmi closed
[cn4:17828] mca: base: close: unloading component pmi
[cn4:17828] [[15336,0],3] grpcomm:base:receive start comm
[cn4:17828] [[15336,0],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn4:17828] [[15336,0],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 15
[cn4:17828] [[15336,0],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 32
[cn4:17828] [[15336,0],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 11
[cn3:21416] mca: base: components_register: registering dstore components
[cn3:21416] mca: base: components_register: found loaded component hash
[cn3:21416] mca: base: components_register: component hash register function successful
[cn3:21416] mca: base: components_open: opening dstore components
[cn3:21416] mca: base: components_open: found loaded component hash
[cn3:21416] mca:dstore:select: checking available component hash
[cn3:21416] mca:dstore:select: Querying component [hash]
[cn3:21416] mca: base: components_register: registering grpcomm components
[cn3:21416] mca: base: components_register: found loaded component bad
[cn3:21416] mca: base: components_register: component bad has no register or open function
[cn3:21416] mca: base: components_register: found loaded component pmi
[cn3:21416] mca: base: components_register: component pmi register function successful
[cn3:21416] mca: base: components_open: opening grpcomm components
[cn3:21416] mca: base: components_open: found loaded component bad
[cn3:21416] mca: base: components_open: component bad open function successful
[cn3:21416] mca: base: components_open: found loaded component pmi
[cn3:21416] mca: base: components_open: component pmi open function successful
[cn3:21416] mca:base:select: Auto-selecting grpcomm components
[cn3:21416] mca:base:select:(grpcomm) Querying component [bad]
[cn3:21416] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn3:21416] mca:base:select:(grpcomm) Querying component [pmi]
[cn3:21416] mca:base:select:(grpcomm) Skipping component [pmi]. Query failed to return a module
[cn3:21416] mca:base:select:(grpcomm) Selected component [bad]
[cn3:21416] mca: base: close: component pmi closed
[cn3:21416] mca: base: close: unloading component pmi
[cn3:21416] [[15336,0],2] grpcomm:base:receive start comm
[cn3:21416] [[15336,0],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn3:21416] [[15336,0],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 15
[cn3:21416] [[15336,0],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 32
[cn3:21416] [[15336,0],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 11
[cn4:17828] [[15336,0],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn4:17828] [[15336,0],3] routed:binomial rank 0 parent 0 me 3 num_procs 5
[cn4:17828] [[15336,0],3] routed:binomial find children of rank 0
[cn4:17828] [[15336,0],3] routed:binomial find children checking peer 1
[cn4:17828] [[15336,0],3] routed:binomial find children computing tree
[cn4:17828] [[15336,0],3] routed:binomial rank 1 parent 0 me 3 num_procs 5
[cn4:17828] [[15336,0],3] routed:binomial find children of rank 1
[cn4:17828] [[15336,0],3] routed:binomial find children checking peer 3
[cn4:17828] [[15336,0],3] routed:binomial find children computing tree
[cn4:17828] [[15336,0],3] routed:binomial rank 3 parent 1 me 3 num_procs 5
[cn4:17828] [[15336,0],3] routed:binomial find children returning found value 1
[cn4:17828] [[15336,0],3] routed:binomial find children returning found value 1
[cn4:17828] [[15336,0],3]: parent 1 num_children 0
[cn4:17828] [[15336,0],3] setting up session dir with
	tmpdir: UNDEF
	host cn4
[cn4:17828] procdir: /tmp/openmpi-sessions-artpol@cn4_0/15336/0/3
[cn4:17828] jobdir: /tmp/openmpi-sessions-artpol@cn4_0/15336/0
[cn4:17828] top: openmpi-sessions-artpol@cn4_0
[cn4:17828] tmp: /tmp
[cn4:17828] sess_dir_cleanup: job session dir does not exist
[cn4:17828] procdir: /tmp/openmpi-sessions-artpol@cn4_0/15336/0/3
[cn4:17828] jobdir: /tmp/openmpi-sessions-artpol@cn4_0/15336/0
[cn4:17828] top: openmpi-sessions-artpol@cn4_0
[cn4:17828] tmp: /tmp
[cn4:17828] [[15336,0],3] routed_binomial: init routes for daemon job [15336,0]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn4:17828] [[15336,0],3] routed_binomial: completed init routes
[cn3:21416] [[15336,0],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn3:21416] [[15336,0],2] routed:binomial rank 0 parent 0 me 2 num_procs 5
[cn3:21416] [[15336,0],2] routed:binomial find children of rank 0
[cn3:21416] [[15336,0],2] routed:binomial find children checking peer 1
[cn3:21416] [[15336,0],2] routed:binomial find children computing tree
[cn3:21416] [[15336,0],2] routed:binomial rank 1 parent 0 me 2 num_procs 5
[cn3:21416] [[15336,0],2] routed:binomial find children of rank 1
[cn3:21416] [[15336,0],2] routed:binomial find children checking peer 3
[cn3:21416] [[15336,0],2] routed:binomial find children computing tree
[cn3:21416] [[15336,0],2] routed:binomial rank 3 parent 1 me 2 num_procs 5
[cn3:21416] [[15336,0],2] routed:binomial find children of rank 3
[cn3:21416] [[15336,0],2] routed:binomial find children checking peer 7
[cn3:21416] [[15336,0],2] routed:binomial find children checking peer 5
[cn3:21416] [[15336,0],2] routed:binomial find children checking peer 2
[cn3:21416] [[15336,0],2] routed:binomial find children computing tree
[cn3:21416] [[15336,0],2] routed:binomial rank 2 parent 0 me 2 num_procs 5
[cn3:21416] [[15336,0],2] routed:binomial find children returning found value 0
[cn3:21416] [[15336,0],2]: parent 0 num_children 0
[cn3:21416] [[15336,0],2] setting up session dir with
	tmpdir: UNDEF
	host cn3
[cn3:21416] procdir: /tmp/openmpi-sessions-artpol@cn3_0/15336/0/2
[cn3:21416] jobdir: /tmp/openmpi-sessions-artpol@cn3_0/15336/0
[cn3:21416] top: openmpi-sessions-artpol@cn3_0
[cn3:21416] tmp: /tmp
[cn3:21416] sess_dir_cleanup: job session dir does not exist
[cn4:17828] [[15336,0],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 3
[cn4:17828] [[15336,0],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 21
[cn4:17828] [[15336,0],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 45
[cn4:17828] [[15336,0],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 46
[cn4:17828] [[15336,0],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 1
Daemon [[15336,0],3] checking in as pid 17828 on host cn4
[cn4:17828] progress: progress_set_yield_when_idle to false
[cn4:17828] progress: set_event_flag setting to 1
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 10
[cn4:17828] [[15336,0],3] orted: up and running - waiting for commands!
[cn4:17828] [[15336,0],3] posting recv
[cn4:17828] [[15336,0],3] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn4:17828] [[15336,0],3] posting recv
[cn4:17828] [[15336,0],3] posting persistent recv on tag 15 for peer [[WILDCARD],WILDCARD]
[cn4:17828] [[15336,0],3] posting recv
[cn4:17828] [[15336,0],3] posting persistent recv on tag 32 for peer [[WILDCARD],WILDCARD]
[cn4:17828] [[15336,0],3] posting recv
[cn4:17828] [[15336,0],3] posting persistent recv on tag 11 for peer [[WILDCARD],WILDCARD]
[cn4:17828] [[15336,0],3] posting recv
[cn4:17828] [[15336,0],3] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn4:17828] [[15336,0],3]: set_addr to uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn4:17828] [[15336,0],3]:set_addr checking if peer [[15336,0],0] is reachable via component tcp
[cn4:17828] [[15336,0],3] oob:tcp: working peer [[15336,0],0] address tcp://192.168.1.1,192.168.10.2:37347
[cn4:17828] [[15336,0],3] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn4:17828] [[15336,0],3] PASSING ADDR 192.168.1.1 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn4:17828] [[15336,0],3]:tcp set addr for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] UNFOUND KERNEL INDEX -13 FOR ADDRESS 192.168.10.2
[cn4:17828] [[15336,0],3]: peer [[15336,0],0] is reachable via component tcp
[cn4:17828] [[15336,0],3] posting recv
[cn4:17828] [[15336,0],3] posting persistent recv on tag 3 for peer [[WILDCARD],WILDCARD]
[cn4:17828] [[15336,0],3] posting recv
[cn4:17828] [[15336,0],3] posting persistent recv on tag 21 for peer [[WILDCARD],WILDCARD]
[cn4:17828] [[15336,0],3] posting recv
[cn4:17828] [[15336,0],3] posting persistent recv on tag 45 for peer [[WILDCARD],WILDCARD]
[cn4:17828] [[15336,0],3] posting recv
[cn4:17828] [[15336,0],3] posting persistent recv on tag 46 for peer [[WILDCARD],WILDCARD]
[cn4:17828] [[15336,0],3] posting recv
[cn4:17828] [[15336,0],3] posting persistent recv on tag 1 for peer [[WILDCARD],WILDCARD]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 10
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:tcp:processing set_peer cmd for interface eth0
[cn4:17828] [[15336,0],3] SET_PEER ADDING PEER [[15336,0],0]
[cn4:17828] [[15336,0],3] set_peer: peer [[15336,0],0] is listening on interface eth0 net 192.168.1.1 port 37347
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:10
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:10
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: initiating connection to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],0]
[cn4:17828] [[15336,0],3] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0
[cn1:14209] [[15336,0],0] mca_oob_tcp_listen_thread: new connection: (16, 11) 192.168.1.4:54524
[cn1:14209] [[15336,0],0] connection_handler: working connection (16, 11) 192.168.1.4:54524
[cn1:14209] [[15336,0],0] accept_connection: 192.168.1.4:54524
[cn1:14209] [[15336,0],0]:tcp:recv:handler called
[cn1:14209] [[15336,0],0] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 16
[cn1:14209] [[15336,0],0] waiting for connect ack from UNKNOWN
[cn1:14209] [[15336,0],0] connect ack received from UNKNOWN
[cn1:14209] [[15336,0],0] connect-ack recvd from UNKNOWN
[cn1:14209] [[15336,0],0] mca_oob_tcp_recv_connect: connection from new peer
[cn1:14209] [[15336,0],0] connect-ack header from [[15336,0],3] is okay
[cn1:14209] [[15336,0],0] waiting for connect ack from [[15336,0],3]
[cn1:14209] [[15336,0],0] connect ack received from [[15336,0],3]
[cn1:14209] [[15336,0],0] connect-ack version from [[15336,0],3] matches ours
[cn1:14209] [[15336,0],0] connect-ack [[15336,0],3] authenticated
[cn1:14209] [[15336,0],0] tcp:peer_accept called for peer [[15336,0],3] in state UNKNOWN on socket 16
[cn1:14209] [[15336,0],0] SEND CONNECT ACK
[cn1:14209] [[15336,0],0] send blocking of 47 bytes to socket 16
[cn1:14209] [[15336,0],0] blocking send complete to socket 16
[cn1:14209] [[15336,0],0]-[[15336,0],3] tcp_peer_connected on socket 16
[cn1:14209] [[15336,0],0] routed_binomial_update: [[15336,0],3] --> [[15336,0],3]
[cn1:14209] [[15336,0],0]-[[15336,0],3] accepted: 192.168.1.1 - 192.168.1.4 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn1:14209] [[15336,0],0] tcp:set_module called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:tcp:peer creating socket to [[15336,0],0]
[cn4:17828] [[15336,0],3] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0 on socket 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] on 192.168.1.1:37347 - 0 retries
[cn4:17828] [[15336,0],3] waiting for connect completion to [[15336,0],0] - activating send event
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler CONNECTING
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:tcp:complete_connect called for peer [[15336,0],0] on socket 9
[cn4:17828] [[15336,0],3] tcp_peer_complete_connect: sending ack to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 97
[cn4:17828] [[15336,0],3] SEND CONNECT ACK
[cn4:17828] fetching data from INTERNAL dstore
[cn4:17828] dstore:hash:fetch: searching for key opal.cred on proc 13889961984
[cn4:17828] dstore_hash:fetch key opal.cred for proc 13889961984 not found
[cn4:17828] [[15336,0],3] send blocking of 47 bytes to socket 9
[cn4:17828] [[15336,0],3] blocking send complete to socket 9
[cn4:17828] [[15336,0],3] tcp_peer_complete_connect: setting read event on connection to [[15336,0],0]
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] RECV CONNECT ACK FROM [[15336,0],0] ON SOCKET 9
[cn4:17828] [[15336,0],3] waiting for connect ack from [[15336,0],0]
[cn4:17828] [[15336,0],3] connect ack received from [[15336,0],0]
[cn4:17828] [[15336,0],3] connect-ack recvd from [[15336,0],0]
[cn4:17828] [[15336,0],3] connect-ack header from [[15336,0],0] is okay
[cn4:17828] [[15336,0],3] waiting for connect ack from [[15336,0],0]
[cn4:17828] [[15336,0],3] connect ack received from [[15336,0],0]
[cn4:17828] [[15336,0],3] connect-ack version from [[15336,0],0] matches ours
[cn4:17828] [[15336,0],3] connect-ack [[15336,0],0] authenticated
[cn4:17828] [[15336,0],3]-[[15336,0],0] tcp_peer_connected on socket 9
[cn4:17828] [[15336,0],3] routed_binomial_update: [[15336,0],0] --> [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] connected: 192.168.1.4 - 192.168.1.1 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn4:17828] [[15336,0],3]:tcp:recv:handler starting send/recv events
[cn4:17828] [[15336,0],3] tcp:set_module called for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 97 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] procdir: /tmp/openmpi-sessions-artpol@cn3_0/15336/0/2
[cn3:21416] jobdir: /tmp/openmpi-sessions-artpol@cn3_0/15336/0
[cn3:21416] top: openmpi-sessions-artpol@cn3_0
[cn3:21416] tmp: /tmp
[cn3:21416] [[15336,0],2] routed_binomial: init routes for daemon job [15336,0]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn3:21416] [[15336,0],2] routed_binomial: completed init routes
[cn3:21416] [[15336,0],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 3
[cn3:21416] [[15336,0],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 21
[cn3:21416] [[15336,0],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 45
[cn3:21416] [[15336,0],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 46
[cn3:21416] [[15336,0],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 1
Daemon [[15336,0],2] checking in as pid 21416 on host cn3
[cn3:21416] progress: progress_set_yield_when_idle to false
[cn3:21416] progress: set_event_flag setting to 1
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 10
[cn3:21416] [[15336,0],2] orted: up and running - waiting for commands!
[cn3:21416] [[15336,0],2] posting recv
[cn3:21416] [[15336,0],2] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn3:21416] [[15336,0],2] posting recv
[cn3:21416] [[15336,0],2] posting persistent recv on tag 15 for peer [[WILDCARD],WILDCARD]
[cn3:21416] [[15336,0],2] posting recv
[cn3:21416] [[15336,0],2] posting persistent recv on tag 32 for peer [[WILDCARD],WILDCARD]
[cn3:21416] [[15336,0],2] posting recv
[cn3:21416] [[15336,0],2] posting persistent recv on tag 11 for peer [[WILDCARD],WILDCARD]
[cn3:21416] [[15336,0],2] posting recv
[cn3:21416] [[15336,0],2] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn3:21416] [[15336,0],2]: set_addr to uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn3:21416] [[15336,0],2]:set_addr checking if peer [[15336,0],0] is reachable via component tcp
[cn3:21416] [[15336,0],2] oob:tcp: working peer [[15336,0],0] address tcp://192.168.1.1,192.168.10.2:37347
[cn3:21416] [[15336,0],2] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn3:21416] [[15336,0],2] PASSING ADDR 192.168.1.1 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn3:21416] [[15336,0],2]:tcp set addr for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] UNFOUND KERNEL INDEX -13 FOR ADDRESS 192.168.10.2
[cn3:21416] [[15336,0],2]: peer [[15336,0],0] is reachable via component tcp
[cn3:21416] [[15336,0],2] posting recv
[cn3:21416] [[15336,0],2] posting persistent recv on tag 3 for peer [[WILDCARD],WILDCARD]
[cn3:21416] [[15336,0],2] posting recv
[cn3:21416] [[15336,0],2] posting persistent recv on tag 21 for peer [[WILDCARD],WILDCARD]
[cn3:21416] [[15336,0],2] posting recv
[cn3:21416] [[15336,0],2] posting persistent recv on tag 45 for peer [[WILDCARD],WILDCARD]
[cn3:21416] [[15336,0],2] posting recv
[cn3:21416] [[15336,0],2] posting persistent recv on tag 46 for peer [[WILDCARD],WILDCARD]
[cn3:21416] [[15336,0],2] posting recv
[cn3:21416] [[15336,0],2] posting persistent recv on tag 1 for peer [[WILDCARD],WILDCARD]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 10
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:tcp:processing set_peer cmd for interface eth0
[cn3:21416] [[15336,0],2] SET_PEER ADDING PEER [[15336,0],0]
[cn3:21416] [[15336,0],2] set_peer: peer [[15336,0],0] is listening on interface eth0 net 192.168.1.1 port 37347
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:10
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:10
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] mca_oob_tcp_listen_thread: new connection: (17, 11) 192.168.1.3:35374
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: initiating connection to [[15336,0],0]
[cn1:14209] [[15336,0],0] connection_handler: working connection (17, 11) 192.168.1.3:35374
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],0]
[cn3:21416] [[15336,0],2] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0
[cn3:21416] [[15336,0],2] oob:tcp:peer creating socket to [[15336,0],0]
[cn1:14209] [[15336,0],0] accept_connection: 192.168.1.3:35374
[cn3:21416] [[15336,0],2] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0 on socket 9
[cn3:21416] [[15336,0],2] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] on 192.168.1.1:37347 - 0 retries
[cn3:21416] [[15336,0],2] waiting for connect completion to [[15336,0],0] - activating send event
[cn1:14209] [[15336,0],0]:tcp:recv:handler called
[cn1:14209] [[15336,0],0] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 17
[cn1:14209] [[15336,0],0] waiting for connect ack from UNKNOWN
[cn1:14209] [[15336,0],0] connect ack received from UNKNOWN
[cn1:14209] [[15336,0],0] connect-ack recvd from UNKNOWN
[cn1:14209] [[15336,0],0] mca_oob_tcp_recv_connect: connection from new peer
[cn1:14209] [[15336,0],0] connect-ack header from [[15336,0],2] is okay
[cn1:14209] [[15336,0],0] waiting for connect ack from [[15336,0],2]
[cn1:14209] [[15336,0],0] connect ack received from [[15336,0],2]
[cn1:14209] [[15336,0],0] connect-ack version from [[15336,0],2] matches ours
[cn1:14209] [[15336,0],0] connect-ack [[15336,0],2] authenticated
[cn1:14209] [[15336,0],0] tcp:peer_accept called for peer [[15336,0],2] in state UNKNOWN on socket 17
[cn1:14209] [[15336,0],0] SEND CONNECT ACK
[cn1:14209] [[15336,0],0] send blocking of 47 bytes to socket 17
[cn1:14209] [[15336,0],0] blocking send complete to socket 17
[cn1:14209] [[15336,0],0]-[[15336,0],2] tcp_peer_connected on socket 17
[cn1:14209] [[15336,0],0] routed_binomial_update: [[15336,0],2] --> [[15336,0],2]
[cn1:14209] [[15336,0],0]-[[15336,0],2] accepted: 192.168.1.1 - 192.168.1.3 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn1:14209] [[15336,0],0] tcp:set_module called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler CONNECTING
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:complete_connect called for peer [[15336,0],0] on socket 9
[cn3:21416] [[15336,0],2] tcp_peer_complete_connect: sending ack to [[15336,0],0]
[cn3:21416] [[15336,0],2] SEND CONNECT ACK
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] fetching data from INTERNAL dstore
[cn3:21416] dstore:hash:fetch: searching for key opal.cred on proc 9594994688
[cn3:21416] dstore_hash:fetch key opal.cred for proc 9594994688 not found
[cn3:21416] [[15336,0],2] send blocking of 47 bytes to socket 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] blocking send complete to socket 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 97
[cn3:21416] [[15336,0],2] tcp_peer_complete_connect: setting read event on connection to [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] RECV CONNECT ACK FROM [[15336,0],0] ON SOCKET 9
[cn3:21416] [[15336,0],2] waiting for connect ack from [[15336,0],0]
[cn3:21416] [[15336,0],2] connect ack received from [[15336,0],0]
[cn3:21416] [[15336,0],2] connect-ack recvd from [[15336,0],0]
[cn3:21416] [[15336,0],2] connect-ack header from [[15336,0],0] is okay
[cn3:21416] [[15336,0],2] waiting for connect ack from [[15336,0],0]
[cn3:21416] [[15336,0],2] connect ack received from [[15336,0],0]
[cn3:21416] [[15336,0],2] connect-ack version from [[15336,0],0] matches ours
[cn3:21416] [[15336,0],2] connect-ack [[15336,0],0] authenticated
[cn3:21416] [[15336,0],2]-[[15336,0],0] tcp_peer_connected on socket 9
[cn3:21416] [[15336,0],2] routed_binomial_update: [[15336,0],0] --> [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] connected: 192.168.1.3 - 192.168.1.1 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn3:21416] [[15336,0],2]:tcp:recv:handler starting send/recv events
[cn3:21416] [[15336,0],2] tcp:set_module called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 97 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 97 BYTES FOR DEST [[15336,0],0] TAG 10
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 97 bytes from [[15336,0],4] for tag 10
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch from daemon [[15336,0],4]
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch from daemon [[15336,0],4] on node cn5
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch attempting to assign daemon [[15336,0],4] to node cn5
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch completed for daemon [[15336,0],4] at contact 1005060096.4;tcp://192.168.1.5:34732
[cn1:14209] [[15336,0],0]: set_addr to uri 1005060096.4;tcp://192.168.1.5:34732
[cn1:14209] [[15336,0],0]:set_addr checking if peer [[15336,0],4] is reachable via component tcp
[cn1:14209] [[15336,0],0] oob:tcp: working peer [[15336,0],4] address tcp://192.168.1.5:34732
[cn1:14209] [[15336,0],0] PEER [[15336,0],4] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn1:14209] [[15336,0],0] PASSING ADDR 192.168.1.5 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn1:14209] [[15336,0],0]:tcp set addr for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]: peer [[15336,0],4] is reachable via component tcp
[cn1:14209] [[15336,0],0]:tcp:processing set_peer cmd for interface eth0
[cn1:14209] [[15336,0],0] set_peer: peer [[15336,0],4] is listening on interface eth0 net 192.168.1.5 port 34732
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 97 BYTES FOR DEST [[15336,0],0] TAG 10
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 97 bytes from [[15336,0],3] for tag 10
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch from daemon [[15336,0],3]
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch from daemon [[15336,0],3] on node cn4
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch attempting to assign daemon [[15336,0],3] to node cn4
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch completed for daemon [[15336,0],3] at contact 1005060096.3;tcp://192.168.1.4:36268
[cn1:14209] [[15336,0],0]: set_addr to uri 1005060096.3;tcp://192.168.1.4:36268
[cn1:14209] [[15336,0],0]:set_addr checking if peer [[15336,0],3] is reachable via component tcp
[cn1:14209] [[15336,0],0] oob:tcp: working peer [[15336,0],3] address tcp://192.168.1.4:36268
[cn1:14209] [[15336,0],0] PEER [[15336,0],3] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn1:14209] [[15336,0],0] PASSING ADDR 192.168.1.4 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn1:14209] [[15336,0],0]:tcp set addr for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]: peer [[15336,0],3] is reachable via component tcp
[cn1:14209] [[15336,0],0]:tcp:processing set_peer cmd for interface eth0
[cn1:14209] [[15336,0],0] set_peer: peer [[15336,0],3] is listening on interface eth0 net 192.168.1.4 port 36268
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 97 BYTES FOR DEST [[15336,0],0] TAG 10
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 97 bytes from [[15336,0],2] for tag 10
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch from daemon [[15336,0],2]
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch from daemon [[15336,0],2] on node cn3
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch attempting to assign daemon [[15336,0],2] to node cn3
[cn1:14209] [[15336,0],0] plm:base:orted_report_launch completed for daemon [[15336,0],2] at contact 1005060096.2;tcp://192.168.1.3:42008
[cn1:14209] [[15336,0],0]: set_addr to uri 1005060096.2;tcp://192.168.1.3:42008
[cn1:14209] [[15336,0],0]:set_addr checking if peer [[15336,0],2] is reachable via component tcp
[cn1:14209] [[15336,0],0] oob:tcp: working peer [[15336,0],2] address tcp://192.168.1.3:42008
[cn1:14209] [[15336,0],0] PEER [[15336,0],2] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn1:14209] [[15336,0],0] PASSING ADDR 192.168.1.3 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn1:14209] [[15336,0],0]:tcp set addr for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]: peer [[15336,0],2] is reachable via component tcp
[cn1:14209] [[15336,0],0]:tcp:processing set_peer cmd for interface eth0
[cn1:14209] [[15336,0],0] set_peer: peer [[15336,0],2] is listening on interface eth0 net 192.168.1.3 port 42008
[cn1:14209] mca:rmaps: mapping job [15336,1]
[cn1:14209] mca:rmaps: creating new map for job [15336,1]
[cn1:14209] mca:rmaps: nprocs 0
[cn1:14209] mca:rmaps mapping not given - using bycore
[cn1:14209] mca:rmaps:ppr: job [15336,1] not using ppr mapper
[cn1:14209] [[15336,0],0] rmaps:seq mapping job [15336,1]
[cn1:14209] mca:rmaps:seq: job [15336,1] not using seq mapper
[cn1:14209] mca:rmaps:resilient: cannot perform initial map of job [15336,1] - no fault groups
[cn1:14209] mca:rmaps:mindist: job [15336,1] not using mindist mapper
[cn1:14209] mca:rmaps:rr: mapping job [15336,1]
[cn1:14209] [[15336,0],0] Starting with 5 nodes in list
[cn1:14209] [[15336,0],0] Filtering thru apps
[cn1:14209] [[15336,0],0] Retained 5 nodes in list
[cn1:14209] [[15336,0],0] node cn1 has 2 slots available
[cn1:14209] [[15336,0],0] node cn2 has 2 slots available
[cn1:14209] [[15336,0],0] node cn3 has 2 slots available
[cn1:14209] [[15336,0],0] node cn4 has 2 slots available
[cn1:14209] [[15336,0],0] node cn5 has 2 slots available
[cn1:14209] AVAILABLE NODES FOR MAPPING:
[cn1:14209]     node: cn1 daemon: 0
[cn1:14209]     node: cn2 daemon: 1
[cn1:14209]     node: cn3 daemon: 2
[cn1:14209]     node: cn4 daemon: 3
[cn1:14209]     node: cn5 daemon: 4
[cn1:14209] [[15336,0],0] Starting bookmark at node cn1
[cn1:14209] [[15336,0],0] Starting at node cn1
[cn1:14209] mca:rmaps:rr: mapping no-span by Core for job [15336,1] slots 10 num_procs 10
[cn1:14209] mca:rmaps:rr: found 8 Core objects on node cn1
[cn1:14209] mca:rmaps:rr: calculated nprocs 2
[cn1:14209] mca:rmaps:rr: assigning nprocs 2
[cn1:14209] mca:rmaps:rr: assigning proc to object 0
[cn1:14209] mca:rmaps:rr: assigning proc to object 1
[cn1:14209] mca:rmaps:rr: found 8 Core objects on node cn2
[cn1:14209] mca:rmaps:rr: calculated nprocs 2
[cn1:14209] mca:rmaps:rr: assigning nprocs 2
[cn1:14209] mca:rmaps:rr: assigning proc to object 0
[cn1:14209] mca:rmaps:rr: assigning proc to object 1
[cn1:14209] mca:rmaps:rr: found 8 Core objects on node cn3
[cn1:14209] mca:rmaps:rr: calculated nprocs 2
[cn1:14209] mca:rmaps:rr: assigning nprocs 2
[cn1:14209] mca:rmaps:rr: assigning proc to object 0
[cn1:14209] mca:rmaps:rr: assigning proc to object 1
[cn1:14209] mca:rmaps:rr: found 8 Core objects on node cn4
[cn1:14209] mca:rmaps:rr: calculated nprocs 2
[cn1:14209] mca:rmaps:rr: assigning nprocs 2
[cn1:14209] mca:rmaps:rr: assigning proc to object 0
[cn1:14209] mca:rmaps:rr: assigning proc to object 1
[cn1:14209] mca:rmaps:rr: found 8 Core objects on node cn5
[cn1:14209] mca:rmaps:rr: calculated nprocs 2
[cn1:14209] mca:rmaps:rr: assigning nprocs 2
[cn1:14209] mca:rmaps:rr: assigning proc to object 0
[cn1:14209] mca:rmaps:rr: assigning proc to object 1
[cn1:14209] mca:rmaps:base: computing vpids by slot for job [15336,1]
[cn1:14209] mca:rmaps:base: assigning rank 0 to node cn1
[cn1:14209] mca:rmaps:base: assigning rank 1 to node cn1
[cn1:14209] mca:rmaps:base: assigning rank 2 to node cn2
[cn1:14209] mca:rmaps:base: assigning rank 3 to node cn2
[cn1:14209] mca:rmaps:base: assigning rank 4 to node cn3
[cn1:14209] mca:rmaps:base: assigning rank 5 to node cn3
[cn1:14209] mca:rmaps:base: assigning rank 6 to node cn4
[cn1:14209] mca:rmaps:base: assigning rank 7 to node cn4
[cn1:14209] mca:rmaps:base: assigning rank 8 to node cn5
[cn1:14209] mca:rmaps:base: assigning rank 9 to node cn5
[cn1:14209] [[15336,0],0] rmaps:base:compute_usage
[cn1:14209] mca:rmaps: compute bindings for job [15336,1] with policy CORE
[cn1:14209] mca:rmaps: bindings for job [15336,1] - bind in place
[cn1:14209] mca:rmaps: bind in place for job [15336,1] with bindings CORE
[cn1:14209] [[15336,0],0] reset_usage: node cn1 has 2 procs on it
[cn1:14209] [[15336,0],0] reset_usage: ignoring proc [[15336,1],0]
[cn1:14209] [[15336,0],0] reset_usage: ignoring proc [[15336,1],1]
[cn1:14209] BINDING PROC [[15336,1],0] TO Core NUMBER 0
[cn1:14209] [[15336,0],0] BOUND PROC [[15336,1],0] TO 0[Core:0] on node cn1
[cn1:14209] BINDING PROC [[15336,1],1] TO Core NUMBER 1
[cn1:14209] [[15336,0],0] BOUND PROC [[15336,1],1] TO 2[Core:1] on node cn1
[cn1:14209] [[15336,0],0] reset_usage: node cn2 has 2 procs on it
[cn1:14209] [[15336,0],0] reset_usage: ignoring proc [[15336,1],2]
[cn1:14209] [[15336,0],0] reset_usage: ignoring proc [[15336,1],3]
[cn1:14209] BINDING PROC [[15336,1],2] TO Core NUMBER 0
[cn1:14209] [[15336,0],0] BOUND PROC [[15336,1],2] TO 0[Core:0] on node cn2
[cn1:14209] BINDING PROC [[15336,1],3] TO Core NUMBER 1
[cn1:14209] [[15336,0],0] BOUND PROC [[15336,1],3] TO 2[Core:1] on node cn2
[cn1:14209] [[15336,0],0] reset_usage: node cn3 has 2 procs on it
[cn1:14209] [[15336,0],0] reset_usage: ignoring proc [[15336,1],4]
[cn1:14209] [[15336,0],0] reset_usage: ignoring proc [[15336,1],5]
[cn1:14209] BINDING PROC [[15336,1],4] TO Core NUMBER 0
[cn1:14209] [[15336,0],0] BOUND PROC [[15336,1],4] TO 0[Core:0] on node cn3
[cn1:14209] BINDING PROC [[15336,1],5] TO Core NUMBER 1
[cn1:14209] [[15336,0],0] BOUND PROC [[15336,1],5] TO 2[Core:1] on node cn3
[cn1:14209] [[15336,0],0] reset_usage: node cn4 has 2 procs on it
[cn1:14209] [[15336,0],0] reset_usage: ignoring proc [[15336,1],6]
[cn1:14209] [[15336,0],0] reset_usage: ignoring proc [[15336,1],7]
[cn1:14209] BINDING PROC [[15336,1],6] TO Core NUMBER 0
[cn1:14209] [[15336,0],0] BOUND PROC [[15336,1],6] TO 0[Core:0] on node cn4
[cn1:14209] BINDING PROC [[15336,1],7] TO Core NUMBER 1
[cn1:14209] [[15336,0],0] BOUND PROC [[15336,1],7] TO 2[Core:1] on node cn4
[cn1:14209] [[15336,0],0] reset_usage: node cn5 has 2 procs on it
[cn1:14209] [[15336,0],0] reset_usage: ignoring proc [[15336,1],8]
[cn1:14209] [[15336,0],0] reset_usage: ignoring proc [[15336,1],9]
[cn1:14209] BINDING PROC [[15336,1],8] TO Core NUMBER 0
[cn1:14209] [[15336,0],0] BOUND PROC [[15336,1],8] TO 0[Core:0] on node cn5
[cn1:14209] BINDING PROC [[15336,1],9] TO Core NUMBER 1
[cn1:14209] [[15336,0],0] BOUND PROC [[15336,1],9] TO 2[Core:1] on node cn5
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 0 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial 0 found child 1
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 1 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 1 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial 1 found child 3
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 3 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 3 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 1
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 3
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 3 parent 1 me 3 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 1
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 1
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 0
[cn1:14209] [[15336,0],0] routed:binomial 0 found child 2
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 1
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 3
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 3 parent 1 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 3
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 7
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 5
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 2
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 2 parent 0 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 0
[cn1:14209] [[15336,0],0] routed:binomial 0 found child 4
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 1
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 3
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 3 parent 1 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 3
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 7
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 5
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 2
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 2 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 2
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 6
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 4
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 4 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 0
[cn1:14209] [[15336,0],0]: parent 0 num_children 3
[cn1:14209] [[15336,0],0]: 	child 1
[cn1:14209] [[15336,0],0]: 		relation 3
[cn1:14209] [[15336,0],0]: 	child 2
[cn1:14209] [[15336,0],0]: 	child 4
[cn1:14209] [[15336,0],0] plm:base:launch_apps for job [15336,1]
[cn1:14209] [[15336,0],0] grpcomm:bad:xcast sent to job [15336,0] tag 1
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 15
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 15
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 15
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 15
[cn1:14209] [[15336,0],0] message received 2668 bytes from [[15336,0],0] for tag 15
[cn1:14209] [[15336,0],0] grpcomm:xcast:recv: with 2668 bytes
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 0 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial 0 found child 1
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 1 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 1 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial 1 found child 3
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 3 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 3 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 1
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 3
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 3 parent 1 me 3 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 1
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 1
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 0
[cn1:14209] [[15336,0],0] routed:binomial 0 found child 2
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 1
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 3
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 3 parent 1 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 3
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 7
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 5
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 2
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 2 parent 0 me 2 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 0
[cn1:14209] [[15336,0],0] routed:binomial 0 found child 4
[cn1:14209] [[15336,0],0] routed:binomial rank 0 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 0
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 1
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 1 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 1
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 3
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 3 parent 1 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 3
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 7
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 5
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 2
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 2 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children of rank 2
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 6
[cn1:14209] [[15336,0],0] routed:binomial find children checking peer 4
[cn1:14209] [[15336,0],0] routed:binomial find children computing tree
[cn1:14209] [[15336,0],0] routed:binomial rank 4 parent 0 me 4 num_procs 5
[cn1:14209] [[15336,0],0] routed:binomial find children returning found value 0
[cn1:14209] [[15336,0],0]: parent 0 num_children 3
[cn1:14209] [[15336,0],0]: 	child 1
[cn1:14209] [[15336,0],0]: 		relation 3
[cn1:14209] [[15336,0],0]: 	child 2
[cn1:14209] [[15336,0],0]: 	child 4
[cn1:14209] [[15336,0],0] routed_binomial: init routes for HNP job [15336,0]
[cn1:14209] [[15336,0],0] rml:base:update:contact:info got uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn1:14209] [[15336,0],0] rml:base:update:contact:info got uri 1005060096.1;tcp://192.168.1.2:51705
[cn1:14209] [[15336,0],0] rml:base:update:contact:info got uri 1005060096.2;tcp://192.168.1.3:42008
[cn1:14209] [[15336,0],0] rml:base:update:contact:info got uri 1005060096.3;tcp://192.168.1.4:36268
[cn1:14209] [[15336,0],0] rml:base:update:contact:info got uri 1005060096.4;tcp://192.168.1.5:34732
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],1]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],1] at tag 15
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],2]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],2] at tag 15
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],4] at tag 15
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 1
[cn1:14209] [[15336,0],0]: set_addr to uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn1:14209] [[15336,0],0]:set_addr peer [[15336,0],0] is me
[cn1:14209] [[15336,0],0]: set_addr to uri 1005060096.1;tcp://192.168.1.2:51705
[cn1:14209] [[15336,0],0]:set_addr checking if peer [[15336,0],1] is reachable via component tcp
[cn1:14209] [[15336,0],0] oob:tcp: working peer [[15336,0],1] address tcp://192.168.1.2:51705
[cn1:14209] [[15336,0],0] PEER [[15336,0],1] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn1:14209] [[15336,0],0] PASSING ADDR 192.168.1.2 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn1:14209] [[15336,0],0]:tcp set addr for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]: peer [[15336,0],1] is reachable via component tcp
[cn1:14209] [[15336,0],0]: set_addr to uri 1005060096.2;tcp://192.168.1.3:42008
[cn1:14209] [[15336,0],0]:set_addr checking if peer [[15336,0],2] is reachable via component tcp
[cn1:14209] [[15336,0],0] oob:tcp: working peer [[15336,0],2] address tcp://192.168.1.3:42008
[cn1:14209] [[15336,0],0] PEER [[15336,0],2] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn1:14209] [[15336,0],0] PASSING ADDR 192.168.1.3 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn1:14209] [[15336,0],0]:tcp set addr for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]: peer [[15336,0],2] is reachable via component tcp
[cn1:14209] [[15336,0],0]: set_addr to uri 1005060096.3;tcp://192.168.1.4:36268
[cn1:14209] [[15336,0],0]:set_addr checking if peer [[15336,0],3] is reachable via component tcp
[cn1:14209] [[15336,0],0] oob:tcp: working peer [[15336,0],3] address tcp://192.168.1.4:36268
[cn1:14209] [[15336,0],0] PEER [[15336,0],3] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn1:14209] [[15336,0],0] PASSING ADDR 192.168.1.4 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn1:14209] [[15336,0],0]:tcp set addr for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]: peer [[15336,0],3] is reachable via component tcp
[cn1:14209] [[15336,0],0]: set_addr to uri 1005060096.4;tcp://192.168.1.5:34732
[cn1:14209] [[15336,0],0]:set_addr checking if peer [[15336,0],4] is reachable via component tcp
[cn1:14209] [[15336,0],0] oob:tcp: working peer [[15336,0],4] address tcp://192.168.1.5:34732
[cn1:14209] [[15336,0],0] PEER [[15336,0],4] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn1:14209] [[15336,0],0] PASSING ADDR 192.168.1.5 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn1:14209] [[15336,0],0]:tcp set addr for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]: peer [[15336,0],4] is reachable via component tcp
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],1] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],2] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],4] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 1
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 1
[cn1:14209] [[15336,0],0]:tcp:processing set_peer cmd for interface eth0
[cn1:14209] [[15336,0],0]:tcp:processing set_peer cmd for interface eth0
[cn1:14209] [[15336,0],0]:tcp:processing set_peer cmd for interface eth0
[cn1:14209] [[15336,0],0]:tcp:processing set_peer cmd for interface eth0
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],1]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],2]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],4]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],4]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],4]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 1
[cn1:14209] [[15336,0],0] message received 2668 bytes from [[15336,0],0] for tag 1
[cn1:14209] [[15336,0],0] orted:comm:process_commands() Processing Command: ORTE_DAEMON_ADD_LOCAL_PROCS
[cn1:14209] [[15336,0],0] orted_cmd: received add_local_procs
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 0
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 1
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 2
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],2] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],2]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],4] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],2]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],2]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 2668 BYTES ON SOCKET 17
[cn1:14209] [[15336,0],0]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],1]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],1]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 2668 BYTES ON SOCKET 14
[cn1:14209] [[15336,0],0]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],4]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 2668 BYTES ON SOCKET 15
[cn1:14209] [[15336,0],0]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 2668
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 2668
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 2668
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 2668 BYTES FOR DEST [[15336,0],4] TAG 15
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] message received 2668 bytes from [[15336,0],0] for tag 15
[cn5:14171] [[15336,0],4] grpcomm:xcast:recv: with 2668 bytes
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 2668 BYTES FOR DEST [[15336,0],2] TAG 15
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 2668 BYTES FOR DEST [[15336,0],1] TAG 15
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] message received 2668 bytes from [[15336,0],0] for tag 15
[cn2:01042] [[15336,0],1] grpcomm:xcast:recv: with 2668 bytes
[cn5:14171] [[15336,0],4] grpcomm:base:xcast updating daemon nidmap
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] message received 2668 bytes from [[15336,0],0] for tag 15
[cn2:01042] [[15336,0],1] grpcomm:base:xcast updating daemon nidmap
[cn3:21416] [[15336,0],2] grpcomm:xcast:recv: with 2668 bytes
[cn3:21416] [[15336,0],2] grpcomm:base:xcast updating daemon nidmap
[cn3:21416] [[15336,0],2] routed:binomial rank 0 parent 0 me 2 num_procs 5
[cn3:21416] [[15336,0],2] routed:binomial find children of rank 0
[cn3:21416] [[15336,0],2] routed:binomial find children checking peer 1
[cn3:21416] [[15336,0],2] routed:binomial find children computing tree
[cn3:21416] [[15336,0],2] routed:binomial rank 1 parent 0 me 2 num_procs 5
[cn3:21416] [[15336,0],2] routed:binomial find children of rank 1
[cn3:21416] [[15336,0],2] routed:binomial find children checking peer 3
[cn3:21416] [[15336,0],2] routed:binomial find children computing tree
[cn3:21416] [[15336,0],2] routed:binomial rank 3 parent 1 me 2 num_procs 5
[cn3:21416] [[15336,0],2] routed:binomial find children of rank 3
[cn3:21416] [[15336,0],2] routed:binomial find children checking peer 7
[cn3:21416] [[15336,0],2] routed:binomial find children checking peer 5
[cn3:21416] [[15336,0],2] routed:binomial find children checking peer 2
[cn3:21416] [[15336,0],2] routed:binomial find children computing tree
[cn3:21416] [[15336,0],2] routed:binomial rank 2 parent 0 me 2 num_procs 5
[cn3:21416] [[15336,0],2] routed:binomial find children returning found value 0
[cn2:01042] [[15336,0],1] routed:binomial rank 0 parent 0 me 1 num_procs 5
[cn2:01042] [[15336,0],1] routed:binomial find children of rank 0
[cn2:01042] [[15336,0],1] routed:binomial find children checking peer 1
[cn2:01042] [[15336,0],1] routed:binomial find children computing tree
[cn2:01042] [[15336,0],1] routed:binomial rank 1 parent 0 me 1 num_procs 5
[cn2:01042] [[15336,0],1] routed:binomial 1 found child 3
[cn2:01042] [[15336,0],1] routed:binomial rank 0 parent 0 me 3 num_procs 5
[cn2:01042] [[15336,0],1] routed:binomial find children of rank 0
[cn2:01042] [[15336,0],1] routed:binomial find children checking peer 1
[cn2:01042] [[15336,0],1] routed:binomial find children computing tree
[cn2:01042] [[15336,0],1] routed:binomial rank 1 parent 0 me 3 num_procs 5
[cn2:01042] [[15336,0],1] routed:binomial find children of rank 1
[cn2:01042] [[15336,0],1] routed:binomial find children checking peer 3
[cn2:01042] [[15336,0],1] routed:binomial find children computing tree
[cn5:14171] [[15336,0],4] routed:binomial rank 0 parent 0 me 4 num_procs 5
[cn5:14171] [[15336,0],4] routed:binomial find children of rank 0
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 1
[cn5:14171] [[15336,0],4] routed:binomial find children computing tree
[cn5:14171] [[15336,0],4] routed:binomial rank 1 parent 0 me 4 num_procs 5
[cn5:14171] [[15336,0],4] routed:binomial find children of rank 1
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 3
[cn5:14171] [[15336,0],4] routed:binomial find children computing tree
[cn5:14171] [[15336,0],4] routed:binomial rank 3 parent 1 me 4 num_procs 5
[cn5:14171] [[15336,0],4] routed:binomial find children of rank 3
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 7
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 5
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 2
[cn5:14171] [[15336,0],4] routed:binomial find children computing tree
[cn3:21416] [[15336,0],2]: parent 0 num_children 0
[cn3:21416] [[15336,0],2] routed_binomial: init routes for daemon job [15336,0]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn3:21416] [[15336,0],2] rml:base:update:contact:info got uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn3:21416] [[15336,0],2] rml:base:update:contact:info got uri 1005060096.1;tcp://192.168.1.2:51705
[cn3:21416] [[15336,0],2] rml:base:update:contact:info got uri 1005060096.2;tcp://192.168.1.3:42008
[cn2:01042] [[15336,0],1] routed:binomial rank 3 parent 1 me 3 num_procs 5
[cn2:01042] [[15336,0],1] routed:binomial find children returning found value 1
[cn2:01042] [[15336,0],1] routed:binomial find children returning found value 1
[cn2:01042] [[15336,0],1] routed:binomial find children returning found value 0
[cn2:01042] [[15336,0],1]: parent 0 num_children 1
[cn2:01042] [[15336,0],1]: 	child 3
[cn2:01042] [[15336,0],1] routed_binomial: init routes for daemon job [15336,0]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn2:01042] [[15336,0],1] rml:base:update:contact:info got uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn2:01042] [[15336,0],1] rml:base:update:contact:info got uri 1005060096.1;tcp://192.168.1.2:51705
[cn2:01042] [[15336,0],1] rml:base:update:contact:info got uri 1005060096.2;tcp://192.168.1.3:42008
[cn2:01042] [[15336,0],1] rml:base:update:contact:info got uri 1005060096.3;tcp://192.168.1.4:36268
[cn5:14171] [[15336,0],4] routed:binomial rank 2 parent 0 me 4 num_procs 5
[cn5:14171] [[15336,0],4] routed:binomial find children of rank 2
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 6
[cn5:14171] [[15336,0],4] routed:binomial find children checking peer 4
[cn5:14171] [[15336,0],4] routed:binomial find children computing tree
[cn5:14171] [[15336,0],4] routed:binomial rank 4 parent 0 me 4 num_procs 5
[cn5:14171] [[15336,0],4] routed:binomial find children returning found value 0
[cn5:14171] [[15336,0],4]: parent 0 num_children 0
[cn5:14171] [[15336,0],4] routed_binomial: init routes for daemon job [15336,0]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn5:14171] [[15336,0],4] rml:base:update:contact:info got uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn5:14171] [[15336,0],4] rml:base:update:contact:info got uri 1005060096.1;tcp://192.168.1.2:51705
[cn5:14171] [[15336,0],4] rml:base:update:contact:info got uri 1005060096.2;tcp://192.168.1.3:42008
[cn3:21416] [[15336,0],2] rml:base:update:contact:info got uri 1005060096.3;tcp://192.168.1.4:36268
[cn3:21416] [[15336,0],2] rml:base:update:contact:info got uri 1005060096.4;tcp://192.168.1.5:34732
[cn3:21416] [[15336,0],2] orte:daemon:send_relay - recipient list is empty!
[cn2:01042] [[15336,0],1] rml:base:update:contact:info got uri 1005060096.4;tcp://192.168.1.5:34732
[cn5:14171] [[15336,0],4] rml:base:update:contact:info got uri 1005060096.3;tcp://192.168.1.4:36268
[cn5:14171] [[15336,0],4] rml:base:update:contact:info got uri 1005060096.4;tcp://192.168.1.5:34732
[cn5:14171] [[15336,0],4] orte:daemon:send_relay - recipient list is empty!
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],4] at tag 1
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],2] at tag 1
[cn3:21416] [[15336,0],2]: set_addr to uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn3:21416] [[15336,0],2]:set_addr checking if peer [[15336,0],0] is reachable via component tcp
[cn3:21416] [[15336,0],2] oob:tcp: working peer [[15336,0],0] address tcp://192.168.1.1,192.168.10.2:37347
[cn2:01042] [[15336,0],1] orte:daemon:send_relay sending relay msg to [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],3] at tag 15
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],1] at tag 1
[cn2:01042] [[15336,0],1]: set_addr to uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn2:01042] [[15336,0],1]:set_addr checking if peer [[15336,0],0] is reachable via component tcp
[cn2:01042] [[15336,0],1] oob:tcp: working peer [[15336,0],0] address tcp://192.168.1.1,192.168.10.2:37347
[cn5:14171] [[15336,0],4]: set_addr to uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn5:14171] [[15336,0],4]:set_addr checking if peer [[15336,0],0] is reachable via component tcp
[cn5:14171] [[15336,0],4] oob:tcp: working peer [[15336,0],0] address tcp://192.168.1.1,192.168.10.2:37347
[cn5:14171] [[15336,0],4] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn5:14171] [[15336,0],4] PASSING ADDR 192.168.1.1 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn5:14171] [[15336,0],4]:tcp set addr for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn3:21416] [[15336,0],2] PASSING ADDR 192.168.1.1 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn3:21416] [[15336,0],2]:tcp set addr for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn2:01042] [[15336,0],1] PASSING ADDR 192.168.1.1 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn5:14171] [[15336,0],4] UNFOUND KERNEL INDEX -13 FOR ADDRESS 192.168.10.2
[cn5:14171] [[15336,0],4]: peer [[15336,0],0] is reachable via component tcp
[cn3:21416] [[15336,0],2] UNFOUND KERNEL INDEX -13 FOR ADDRESS 192.168.10.2
[cn3:21416] [[15336,0],2]: peer [[15336,0],0] is reachable via component tcp
[cn2:01042] [[15336,0],1]:tcp set addr for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]: set_addr to uri 1005060096.1;tcp://192.168.1.2:51705
[cn5:14171] [[15336,0],4]:set_addr checking if peer [[15336,0],1] is reachable via component tcp
[cn5:14171] [[15336,0],4] oob:tcp: working peer [[15336,0],1] address tcp://192.168.1.2:51705
[cn3:21416] [[15336,0],2]: set_addr to uri 1005060096.1;tcp://192.168.1.2:51705
[cn3:21416] [[15336,0],2]:set_addr checking if peer [[15336,0],1] is reachable via component tcp
[cn3:21416] [[15336,0],2] oob:tcp: working peer [[15336,0],1] address tcp://192.168.1.2:51705
[cn2:01042] [[15336,0],1] UNFOUND KERNEL INDEX -13 FOR ADDRESS 192.168.10.2
[cn2:01042] [[15336,0],1]: peer [[15336,0],0] is reachable via component tcp
[cn5:14171] [[15336,0],4] PEER [[15336,0],1] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn5:14171] [[15336,0],4] PASSING ADDR 192.168.1.2 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn3:21416] [[15336,0],2] PEER [[15336,0],1] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn2:01042] [[15336,0],1]: set_addr to uri 1005060096.1;tcp://192.168.1.2:51705
[cn2:01042] [[15336,0],1]:set_addr peer [[15336,0],1] is me
[cn2:01042] [[15336,0],1]: set_addr to uri 1005060096.2;tcp://192.168.1.3:42008
[cn5:14171] [[15336,0],4]:tcp set addr for peer [[15336,0],1]
[cn5:14171] [[15336,0],4]: peer [[15336,0],1] is reachable via component tcp
[cn5:14171] [[15336,0],4]: set_addr to uri 1005060096.2;tcp://192.168.1.3:42008
[cn3:21416] [[15336,0],2] PASSING ADDR 192.168.1.2 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn3:21416] [[15336,0],2]:tcp set addr for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:set_addr checking if peer [[15336,0],2] is reachable via component tcp
[cn2:01042] [[15336,0],1] oob:tcp: working peer [[15336,0],2] address tcp://192.168.1.3:42008
[cn3:21416] [[15336,0],2]: peer [[15336,0],1] is reachable via component tcp
[cn3:21416] [[15336,0],2]: set_addr to uri 1005060096.2;tcp://192.168.1.3:42008
[cn3:21416] [[15336,0],2]:set_addr peer [[15336,0],2] is me
[cn3:21416] [[15336,0],2]: set_addr to uri 1005060096.3;tcp://192.168.1.4:36268
[cn2:01042] [[15336,0],1] PEER [[15336,0],2] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn3:21416] [[15336,0],2]:set_addr checking if peer [[15336,0],3] is reachable via component tcp
[cn3:21416] [[15336,0],2] oob:tcp: working peer [[15336,0],3] address tcp://192.168.1.4:36268
[cn2:01042] [[15336,0],1] PASSING ADDR 192.168.1.3 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn2:01042] [[15336,0],1]:tcp set addr for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] PEER [[15336,0],3] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn3:21416] [[15336,0],2] PASSING ADDR 192.168.1.4 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn3:21416] [[15336,0],2]:tcp set addr for peer [[15336,0],3]
[cn3:21416] [[15336,0],2]: peer [[15336,0],3] is reachable via component tcp
[cn3:21416] [[15336,0],2]: set_addr to uri 1005060096.4;tcp://192.168.1.5:34732
[cn3:21416] [[15336,0],2]:set_addr checking if peer [[15336,0],4] is reachable via component tcp
[cn3:21416] [[15336,0],2] oob:tcp: working peer [[15336,0],4] address tcp://192.168.1.5:34732
[cn2:01042] [[15336,0],1]: peer [[15336,0],2] is reachable via component tcp
[cn2:01042] [[15336,0],1]: set_addr to uri 1005060096.3;tcp://192.168.1.4:36268
[cn3:21416] [[15336,0],2] PEER [[15336,0],4] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn2:01042] [[15336,0],1]:set_addr checking if peer [[15336,0],3] is reachable via component tcp
[cn2:01042] [[15336,0],1] oob:tcp: working peer [[15336,0],3] address tcp://192.168.1.4:36268
[cn3:21416] [[15336,0],2] PASSING ADDR 192.168.1.5 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn3:21416] [[15336,0],2]:tcp set addr for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] PEER [[15336,0],3] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn3:21416] [[15336,0],2]: peer [[15336,0],4] is reachable via component tcp
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],2] at tag 1
[cn3:21416] [[15336,0],2] rml_send_iovec_to_self at tag 1
[cn2:01042] [[15336,0],1] PASSING ADDR 192.168.1.4 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn2:01042] [[15336,0],1]:tcp set addr for peer [[15336,0],3]
[cn3:21416] [[15336,0],2]:tcp:processing set_peer cmd for interface eth0
[cn2:01042] [[15336,0],1]: peer [[15336,0],3] is reachable via component tcp
[cn2:01042] [[15336,0],1]: set_addr to uri 1005060096.4;tcp://192.168.1.5:34732
[cn2:01042] [[15336,0],1]:set_addr checking if peer [[15336,0],4] is reachable via component tcp
[cn2:01042] [[15336,0],1] oob:tcp: working peer [[15336,0],4] address tcp://192.168.1.5:34732
[cn2:01042] [[15336,0],1] PEER [[15336,0],4] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn5:14171] [[15336,0],4]:set_addr checking if peer [[15336,0],2] is reachable via component tcp
[cn5:14171] [[15336,0],4] oob:tcp: working peer [[15336,0],2] address tcp://192.168.1.3:42008
[cn2:01042] [[15336,0],1] PASSING ADDR 192.168.1.5 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn2:01042] [[15336,0],1]:tcp set addr for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] PEER [[15336,0],2] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn5:14171] [[15336,0],4] PASSING ADDR 192.168.1.3 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn3:21416] [[15336,0],2]:tcp:processing set_peer cmd for interface eth0
[cn3:21416] [[15336,0],2] SET_PEER ADDING PEER [[15336,0],1]
[cn2:01042] [[15336,0],1]: peer [[15336,0],4] is reachable via component tcp
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],3] at tag 15
[cn5:14171] [[15336,0],4]:tcp set addr for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]: peer [[15336,0],2] is reachable via component tcp
[cn3:21416] [[15336,0],2] set_peer: peer [[15336,0],1] is listening on interface eth0 net 192.168.1.2 port 51705
[cn3:21416] [[15336,0],2]:tcp:processing set_peer cmd for interface eth0
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],1] at tag 1
[cn5:14171] [[15336,0],4]: set_addr to uri 1005060096.3;tcp://192.168.1.4:36268
[cn3:21416] [[15336,0],2] SET_PEER ADDING PEER [[15336,0],3]
[cn3:21416] [[15336,0],2] set_peer: peer [[15336,0],3] is listening on interface eth0 net 192.168.1.4 port 36268
[cn3:21416] [[15336,0],2]:tcp:processing set_peer cmd for interface eth0
[cn3:21416] [[15336,0],2] SET_PEER ADDING PEER [[15336,0],4]
[cn2:01042] [[15336,0],1] rml_send_iovec_to_self at tag 1
[cn2:01042] [[15336,0],1]:tcp:processing set_peer cmd for interface eth0
[cn5:14171] [[15336,0],4]:set_addr checking if peer [[15336,0],3] is reachable via component tcp
[cn5:14171] [[15336,0],4] oob:tcp: working peer [[15336,0],3] address tcp://192.168.1.4:36268
[cn3:21416] [[15336,0],2] set_peer: peer [[15336,0],4] is listening on interface eth0 net 192.168.1.5 port 34732
[cn3:21416] [[15336,0],2] rml_send_to_self callback executing for tag 1
[cn3:21416] [[15336,0],2] message received 2668 bytes from [[15336,0],2] for tag 1
[cn2:01042] [[15336,0],1]:tcp:processing set_peer cmd for interface eth0
[cn2:01042] [[15336,0],1] SET_PEER ADDING PEER [[15336,0],2]
[cn5:14171] [[15336,0],4] PEER [[15336,0],3] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn5:14171] [[15336,0],4] PASSING ADDR 192.168.1.4 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn3:21416] [[15336,0],2] orted:comm:process_commands() Processing Command: ORTE_DAEMON_ADD_LOCAL_PROCS
[cn3:21416] [[15336,0],2] orted_cmd: received add_local_procs
[cn2:01042] [[15336,0],1] set_peer: peer [[15336,0],2] is listening on interface eth0 net 192.168.1.3 port 42008
[cn2:01042] [[15336,0],1]:tcp:processing set_peer cmd for interface eth0
[cn2:01042] [[15336,0],1] SET_PEER ADDING PEER [[15336,0],3]
[cn5:14171] [[15336,0],4]:tcp set addr for peer [[15336,0],3]
[cn5:14171] [[15336,0],4]: peer [[15336,0],3] is reachable via component tcp
[cn2:01042] [[15336,0],1] set_peer: peer [[15336,0],3] is listening on interface eth0 net 192.168.1.4 port 36268
[cn2:01042] [[15336,0],1]:tcp:processing set_peer cmd for interface eth0
[cn5:14171] [[15336,0],4]: set_addr to uri 1005060096.4;tcp://192.168.1.5:34732
[cn5:14171] [[15336,0],4]:set_addr peer [[15336,0],4] is me
[cn2:01042] [[15336,0],1] SET_PEER ADDING PEER [[15336,0],4]
[cn2:01042] [[15336,0],1] set_peer: peer [[15336,0],4] is listening on interface eth0 net 192.168.1.5 port 34732
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],3]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],4] at tag 1
[cn5:14171] [[15336,0],4] rml_send_iovec_to_self at tag 1
[cn5:14171] [[15336,0],4]:tcp:processing set_peer cmd for interface eth0
[cn5:14171] [[15336,0],4]:tcp:processing set_peer cmd for interface eth0
[cn5:14171] [[15336,0],4] SET_PEER ADDING PEER [[15336,0],1]
[cn5:14171] [[15336,0],4] set_peer: peer [[15336,0],1] is listening on interface eth0 net 192.168.1.2 port 51705
[cn5:14171] [[15336,0],4]:tcp:processing set_peer cmd for interface eth0
[cn5:14171] [[15336,0],4] SET_PEER ADDING PEER [[15336,0],2]
[cn5:14171] [[15336,0],4] set_peer: peer [[15336,0],2] is listening on interface eth0 net 192.168.1.3 port 42008
[cn5:14171] [[15336,0],4]:tcp:processing set_peer cmd for interface eth0
[cn5:14171] [[15336,0],4] SET_PEER ADDING PEER [[15336,0],3]
[cn5:14171] [[15336,0],4] set_peer: peer [[15336,0],3] is listening on interface eth0 net 192.168.1.4 port 36268
[cn5:14171] [[15336,0],4] rml_send_to_self callback executing for tag 1
[cn5:14171] [[15336,0],4] message received 2668 bytes from [[15336,0],4] for tag 1
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],3]:15
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],3]
[cn5:14171] [[15336,0],4] orted:comm:process_commands() Processing Command: ORTE_DAEMON_ADD_LOCAL_PROCS
[cn5:14171] [[15336,0],4] orted_cmd: received add_local_procs
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_to_self callback executing for tag 1
[cn2:01042] [[15336,0],1] message received 2668 bytes from [[15336,0],1] for tag 1
[cn2:01042] [[15336,0],1] orted:comm:process_commands() Processing Command: ORTE_DAEMON_ADD_LOCAL_PROCS
[cn2:01042] [[15336,0],1] orted_cmd: received add_local_procs
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 0
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 1
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 2
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:15
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: initiating connection to [[15336,0],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],3]
[cn2:01042] [[15336,0],1] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],3] via interface eth0
[cn2:01042] [[15336,0],1] oob:tcp:peer creating socket to [[15336,0],3]
[cn2:01042] [[15336,0],1] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],3] via interface eth0 on socket 10
[cn2:01042] [[15336,0],1] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],3] on 192.168.1.4:36268 - 0 retries
[cn2:01042] [[15336,0],1] waiting for connect completion to [[15336,0],3] - activating send event
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 0
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 1
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 2
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 0
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 1
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 2
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] connection_event_handler: working connection (10, 115) 192.168.1.2:37004
[cn4:17828] [[15336,0],3] accept_connection: 192.168.1.2:37004
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 5
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 117 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 5
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler CONNECTING
[cn2:01042] [[15336,0],1]:tcp:complete_connect called for peer [[15336,0],3] on socket 10
[cn2:01042] [[15336,0],1] tcp_peer_complete_connect: sending ack to [[15336,0],3]
[cn2:01042] [[15336,0],1] SEND CONNECT ACK
[cn2:01042] [[15336,0],1] send blocking of 47 bytes to socket 10
[cn2:01042] [[15336,0],1] blocking send complete to socket 10
[cn2:01042] [[15336,0],1] tcp_peer_complete_connect: setting read event on connection to [[15336,0],3]
[cn4:17828] [[15336,0],3]:tcp:recv:handler called
[cn4:17828] [[15336,0],3] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 10
[cn4:17828] [[15336,0],3] waiting for connect ack from UNKNOWN
[cn4:17828] [[15336,0],3] connect ack received from UNKNOWN
[cn4:17828] [[15336,0],3] connect-ack recvd from UNKNOWN
[cn4:17828] [[15336,0],3] mca_oob_tcp_recv_connect: connection from new peer
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 117 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] connect-ack header from [[15336,0],1] is okay
[cn4:17828] [[15336,0],3] waiting for connect ack from [[15336,0],1]
[cn4:17828] [[15336,0],3] connect ack received from [[15336,0],1]
[cn4:17828] [[15336,0],3] connect-ack version from [[15336,0],1] matches ours
[cn4:17828] [[15336,0],3] connect-ack [[15336,0],1] authenticated
[cn4:17828] [[15336,0],3] tcp:peer_accept called for peer [[15336,0],1] in state UNKNOWN on socket 10
[cn4:17828] [[15336,0],3] SEND CONNECT ACK
[cn4:17828] [[15336,0],3] send blocking of 47 bytes to socket 10
[cn4:17828] [[15336,0],3] blocking send complete to socket 10
[cn4:17828] [[15336,0],3]-[[15336,0],1] tcp_peer_connected on socket 10
[cn4:17828] [[15336,0],3] routed_binomial_update: [[15336,0],1] --> [[15336,0],1]
[cn4:17828] [[15336,0],3]-[[15336,0],1] accepted: 192.168.1.4 - 192.168.1.2 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn4:17828] [[15336,0],3] tcp:set_module called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] RECV CONNECT ACK FROM [[15336,0],3] ON SOCKET 10
[cn2:01042] [[15336,0],1] waiting for connect ack from [[15336,0],3]
[cn2:01042] [[15336,0],1] connect ack received from [[15336,0],3]
[cn2:01042] [[15336,0],1] connect-ack recvd from [[15336,0],3]
[cn2:01042] [[15336,0],1] connect-ack header from [[15336,0],3] is okay
[cn2:01042] [[15336,0],1] waiting for connect ack from [[15336,0],3]
[cn2:01042] [[15336,0],1] connect ack received from [[15336,0],3]
[cn2:01042] [[15336,0],1] connect-ack version from [[15336,0],3] matches ours
[cn2:01042] [[15336,0],1] connect-ack [[15336,0],3] authenticated
[cn2:01042] [[15336,0],1]-[[15336,0],3] tcp_peer_connected on socket 10
[cn2:01042] [[15336,0],1] routed_binomial_update: [[15336,0],3] --> [[15336,0],3]
[cn2:01042] [[15336,0],1]-[[15336,0],3] connected: 192.168.1.2 - 192.168.1.4 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler starting send/recv events
[cn2:01042] [[15336,0],1] tcp:set_module called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],3]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 2668 BYTES ON SOCKET 10
[cn2:01042] [[15336,0],1]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 2668
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2668 BYTES FOR DEST [[15336,0],3] TAG 15
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] message received 2668 bytes from [[15336,0],1] for tag 15
[cn4:17828] [[15336,0],3] grpcomm:xcast:recv: with 2668 bytes
[cn4:17828] [[15336,0],3] grpcomm:base:xcast updating daemon nidmap
[cn4:17828] [[15336,0],3] routed:binomial rank 0 parent 0 me 3 num_procs 5
[cn4:17828] [[15336,0],3] routed:binomial find children of rank 0
[cn4:17828] [[15336,0],3] routed:binomial find children checking peer 1
[cn4:17828] [[15336,0],3] routed:binomial find children computing tree
[cn4:17828] [[15336,0],3] routed:binomial rank 1 parent 0 me 3 num_procs 5
[cn4:17828] [[15336,0],3] routed:binomial find children of rank 1
[cn4:17828] [[15336,0],3] routed:binomial find children checking peer 3
[cn4:17828] [[15336,0],3] routed:binomial find children computing tree
[cn4:17828] [[15336,0],3] routed:binomial rank 3 parent 1 me 3 num_procs 5
[cn4:17828] [[15336,0],3] routed:binomial find children returning found value 1
[cn4:17828] [[15336,0],3] routed:binomial find children returning found value 1
[cn4:17828] [[15336,0],3]: parent 1 num_children 0
[cn4:17828] [[15336,0],3] routed_binomial: init routes for daemon job [15336,0]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn4:17828] [[15336,0],3] rml:base:update:contact:info got uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn4:17828] [[15336,0],3] rml:base:update:contact:info got uri 1005060096.1;tcp://192.168.1.2:51705
[cn4:17828] [[15336,0],3] rml:base:update:contact:info got uri 1005060096.2;tcp://192.168.1.3:42008
[cn4:17828] [[15336,0],3] rml:base:update:contact:info got uri 1005060096.3;tcp://192.168.1.4:36268
[cn4:17828] [[15336,0],3] rml:base:update:contact:info got uri 1005060096.4;tcp://192.168.1.5:34732
[cn4:17828] [[15336,0],3] orte:daemon:send_relay - recipient list is empty!
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],3] at tag 1
[cn4:17828] [[15336,0],3]: set_addr to uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn4:17828] [[15336,0],3]:set_addr checking if peer [[15336,0],0] is reachable via component tcp
[cn4:17828] [[15336,0],3] oob:tcp: working peer [[15336,0],0] address tcp://192.168.1.1,192.168.10.2:37347
[cn4:17828] [[15336,0],3] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn4:17828] [[15336,0],3] PASSING ADDR 192.168.1.1 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn4:17828] [[15336,0],3]:tcp set addr for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] UNFOUND KERNEL INDEX -13 FOR ADDRESS 192.168.10.2
[cn4:17828] [[15336,0],3]: peer [[15336,0],0] is reachable via component tcp
[cn4:17828] [[15336,0],3]: set_addr to uri 1005060096.1;tcp://192.168.1.2:51705
[cn4:17828] [[15336,0],3]:set_addr checking if peer [[15336,0],1] is reachable via component tcp
[cn4:17828] [[15336,0],3] oob:tcp: working peer [[15336,0],1] address tcp://192.168.1.2:51705
[cn4:17828] [[15336,0],3] PEER [[15336,0],1] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn4:17828] [[15336,0],3] PASSING ADDR 192.168.1.2 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn4:17828] [[15336,0],3]:tcp set addr for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]: peer [[15336,0],1] is reachable via component tcp
[cn4:17828] [[15336,0],3]: set_addr to uri 1005060096.2;tcp://192.168.1.3:42008
[cn4:17828] [[15336,0],3]:set_addr checking if peer [[15336,0],2] is reachable via component tcp
[cn4:17828] [[15336,0],3] oob:tcp: working peer [[15336,0],2] address tcp://192.168.1.3:42008
[cn4:17828] [[15336,0],3] PEER [[15336,0],2] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn4:17828] [[15336,0],3] PASSING ADDR 192.168.1.3 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn4:17828] [[15336,0],3]:tcp set addr for peer [[15336,0],2]
[cn4:17828] [[15336,0],3]: peer [[15336,0],2] is reachable via component tcp
[cn4:17828] [[15336,0],3]: set_addr to uri 1005060096.3;tcp://192.168.1.4:36268
[cn4:17828] [[15336,0],3]:set_addr peer [[15336,0],3] is me
[cn4:17828] [[15336,0],3]: set_addr to uri 1005060096.4;tcp://192.168.1.5:34732
[cn4:17828] [[15336,0],3]:set_addr checking if peer [[15336,0],4] is reachable via component tcp
[cn4:17828] [[15336,0],3] oob:tcp: working peer [[15336,0],4] address tcp://192.168.1.5:34732
[cn4:17828] [[15336,0],3] PEER [[15336,0],4] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn4:17828] [[15336,0],3] PASSING ADDR 192.168.1.5 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn4:17828] [[15336,0],3]:tcp set addr for peer [[15336,0],4]
[cn4:17828] [[15336,0],3]: peer [[15336,0],4] is reachable via component tcp
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],3] at tag 1
[cn4:17828] [[15336,0],3] rml_send_iovec_to_self at tag 1
[cn4:17828] [[15336,0],3]:tcp:processing set_peer cmd for interface eth0
[cn4:17828] [[15336,0],3]:tcp:processing set_peer cmd for interface eth0
[cn4:17828] [[15336,0],3] set_peer: peer [[15336,0],1] is listening on interface eth0 net 192.168.1.2 port 51705
[cn4:17828] [[15336,0],3]:tcp:processing set_peer cmd for interface eth0
[cn4:17828] [[15336,0],3] SET_PEER ADDING PEER [[15336,0],2]
[cn4:17828] [[15336,0],3] set_peer: peer [[15336,0],2] is listening on interface eth0 net 192.168.1.3 port 42008
[cn4:17828] [[15336,0],3]:tcp:processing set_peer cmd for interface eth0
[cn4:17828] [[15336,0],3] SET_PEER ADDING PEER [[15336,0],4]
[cn4:17828] [[15336,0],3] set_peer: peer [[15336,0],4] is listening on interface eth0 net 192.168.1.5 port 34732
[cn4:17828] [[15336,0],3] rml_send_to_self callback executing for tag 1
[cn4:17828] [[15336,0],3] message received 2668 bytes from [[15336,0],3] for tag 1
[cn4:17828] [[15336,0],3] orted:comm:process_commands() Processing Command: ORTE_DAEMON_ADD_LOCAL_PROCS
[cn4:17828] [[15336,0],3] orted_cmd: received add_local_procs
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 5
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 0
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 1
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 2
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 117 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 311 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 311 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 81 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 90 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 87 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 81 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 260 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 311 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 5
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 117 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 779 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 263 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 311 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 500 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 76 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 192 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 192 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 546 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 546 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 192 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 546 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 779 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 81 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 260 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 311 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 172 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 117 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 401 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 462 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 231 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 84 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 537 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1317 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 537 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 117 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 401 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 276 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 443 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 717 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1256 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 263 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 546 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 733 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 76 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 192 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 304 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 520 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 276 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 532 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1280 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 172 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 108 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 117 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 401 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 462 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 285 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 520 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 172 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1135 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 488 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 374 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 216 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 187 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 263 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 304 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 105 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 172 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1253 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 117 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 537 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 401 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 88 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 499 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1060 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 247 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 948 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2106 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 520 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 388 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] connection_event_handler: working connection (12, 0) 192.168.1.5:45213
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] accept_connection: 192.168.1.5:45213
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1110 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 84 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:tcp:recv:handler called
[cn5:14171] [[15336,0],4] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 12
[cn5:14171] [[15336,0],4] waiting for connect ack from UNKNOWN
[cn5:14171] [[15336,0],4] connect ack received from UNKNOWN
[cn5:14171] [[15336,0],4] connect-ack recvd from UNKNOWN
[cn5:14171] [[15336,0],4] mca_oob_tcp_recv_connect: connection from new peer
[cn5:14171] [[15336,0],4] connect-ack header from [[15336,1],8] is okay
[cn5:14171] [[15336,0],4] waiting for connect ack from [[15336,1],8]
[cn5:14171] [[15336,0],4] connect ack received from [[15336,1],8]
[cn5:14171] [[15336,0],4] connect-ack version from [[15336,1],8] matches ours
[cn5:14171] [[15336,0],4] connect-ack [[15336,1],8] authenticated
[cn5:14171] [[15336,0],4] tcp:peer_accept called for peer [[15336,1],8] in state UNKNOWN on socket 12
[cn5:14171] [[15336,0],4] SEND CONNECT ACK
[cn5:14171] [[15336,0],4] send blocking of 47 bytes to socket 12
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] blocking send complete to socket 12
[cn5:14171] [[15336,0],4]-[[15336,1],8] tcp_peer_connected on socket 12
[cn5:14171] [[15336,0],4] routed_binomial_update: [[15336,1],8] --> [[15336,1],8]
[cn5:14171] [[15336,0],4]-[[15336,1],8] accepted: 192.168.1.5 - 192.168.1.5 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 575 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:set_module called for peer [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],8]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 62
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 810 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 206 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1462 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 709 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 960 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 263 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 452 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 304 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1297 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] connection_event_handler: working connection (13, 0) 192.168.1.2:48028
[cn2:01042] [[15336,0],1] accept_connection: 192.168.1.2:48028
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 689 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:tcp:recv:handler called
[cn2:01042] [[15336,0],1] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 13
[cn2:01042] [[15336,0],1] waiting for connect ack from UNKNOWN
[cn2:01042] [[15336,0],1] connect ack received from UNKNOWN
[cn2:01042] [[15336,0],1] connect-ack recvd from UNKNOWN
[cn2:01042] [[15336,0],1] mca_oob_tcp_recv_connect: connection from new peer
[cn2:01042] [[15336,0],1] connect-ack header from [[15336,1],2] is okay
[cn2:01042] [[15336,0],1] waiting for connect ack from [[15336,1],2]
[cn2:01042] [[15336,0],1] connect ack received from [[15336,1],2]
[cn2:01042] [[15336,0],1] connect-ack version from [[15336,1],2] matches ours
[cn2:01042] [[15336,0],1] connect-ack [[15336,1],2] authenticated
[cn2:01042] [[15336,0],1] tcp:peer_accept called for peer [[15336,1],2] in state UNKNOWN on socket 13
[cn2:01042] [[15336,0],1] SEND CONNECT ACK
[cn2:01042] [[15336,0],1] send blocking of 47 bytes to socket 13
[cn2:01042] [[15336,0],1] blocking send complete to socket 13
[cn2:01042] [[15336,0],1]-[[15336,1],2] tcp_peer_connected on socket 13
[cn2:01042] [[15336,0],1] routed_binomial_update: [[15336,1],2] --> [[15336,1],2]
[cn2:01042] [[15336,0],1]-[[15336,1],2] accepted: 192.168.1.2 - 192.168.1.2 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 477 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:set_module called for peer [[15336,1],2]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],2]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 62
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 459 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1217 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 275 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 276 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 267 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 738 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 443 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1256 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 190 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 276 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 160 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 289 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 443 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 104 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1256 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 228 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 691 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 102 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1197 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 118 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 725 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] connection_event_handler: working connection (14, 11) 192.168.1.2:48029
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] accept_connection: 192.168.1.2:48029
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 286 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 172 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:tcp:recv:handler called
[cn2:01042] [[15336,0],1] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 14
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] waiting for connect ack from UNKNOWN
[cn2:01042] [[15336,0],1] connect ack received from UNKNOWN
[cn2:01042] [[15336,0],1] connect-ack recvd from UNKNOWN
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] mca_oob_tcp_recv_connect: connection from new peer
[cn2:01042] [[15336,0],1] connect-ack header from [[15336,1],3] is okay
[cn2:01042] [[15336,0],1] waiting for connect ack from [[15336,1],3]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] connect ack received from [[15336,1],3]
[cn2:01042] [[15336,0],1] connect-ack version from [[15336,1],3] matches ours
[cn2:01042] [[15336,0],1] connect-ack [[15336,1],3] authenticated
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:peer_accept called for peer [[15336,1],3] in state UNKNOWN on socket 14
[cn2:01042] [[15336,0],1] SEND CONNECT ACK
[cn2:01042] [[15336,0],1] send blocking of 47 bytes to socket 14
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] blocking send complete to socket 14
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,1],3] tcp_peer_connected on socket 14
[cn2:01042] [[15336,0],1] routed_binomial_update: [[15336,1],3] --> [[15336,1],3]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 104 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,1],3] accepted: 192.168.1.2 - 192.168.1.2 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 106 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 374 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 117 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1289 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:set_module called for peer [[15336,1],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 401 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 676 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 217 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 424 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 126 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 103 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 136 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 443 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 62
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1144 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1256 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 275 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 267 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 117 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 105 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 738 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 192 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 401 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 738 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 88 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 190 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 190 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 247 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 442 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 104 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 263 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 160 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 102 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 304 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 289 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 419 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 336 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 118 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 739 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 876 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1253 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] connection_event_handler: working connection (13, 11) 192.168.1.5:45214
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] accept_connection: 192.168.1.5:45214
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 286 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1586 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:tcp:recv:handler called
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 13
[cn5:14171] [[15336,0],4] waiting for connect ack from UNKNOWN
[cn5:14171] [[15336,0],4] connect ack received from UNKNOWN
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] connect-ack recvd from UNKNOWN
[cn5:14171] [[15336,0],4] mca_oob_tcp_recv_connect: connection from new peer
[cn5:14171] [[15336,0],4] connect-ack header from [[15336,1],9] is okay
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] waiting for connect ack from [[15336,1],9]
[cn5:14171] [[15336,0],4] connect ack received from [[15336,1],9]
[cn5:14171] [[15336,0],4] connect-ack version from [[15336,1],9] matches ours
[cn5:14171] [[15336,0],4] connect-ack [[15336,1],9] authenticated
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4] tcp:peer_accept called for peer [[15336,1],9] in state UNKNOWN on socket 13
[cn5:14171] [[15336,0],4] SEND CONNECT ACK
[cn5:14171] [[15336,0],4] send blocking of 47 bytes to socket 13
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] blocking send complete to socket 13
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,1],9] tcp_peer_connected on socket 13
[cn5:14171] [[15336,0],4] routed_binomial_update: [[15336,1],9] --> [[15336,1],9]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 110 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]-[[15336,1],9] accepted: 192.168.1.5 - 192.168.1.5 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 459 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:set_module called for peer [[15336,1],9]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 401 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 875 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 810 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],9]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 62
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 95 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1049 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] connection_event_handler: working connection (12, 0) 192.168.1.3:40989
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1462 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] accept_connection: 192.168.1.3:40989
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 611 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler called
[cn3:21416] [[15336,0],2] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 12
[cn3:21416] [[15336,0],2] waiting for connect ack from UNKNOWN
[cn3:21416] [[15336,0],2] connect ack received from UNKNOWN
[cn3:21416] [[15336,0],2] connect-ack recvd from UNKNOWN
[cn3:21416] [[15336,0],2] mca_oob_tcp_recv_connect: connection from new peer
[cn3:21416] [[15336,0],2] connect-ack header from [[15336,1],4] is okay
[cn3:21416] [[15336,0],2] waiting for connect ack from [[15336,1],4]
[cn3:21416] [[15336,0],2] connect ack received from [[15336,1],4]
[cn3:21416] [[15336,0],2] connect-ack version from [[15336,1],4] matches ours
[cn3:21416] [[15336,0],2] connect-ack [[15336,1],4] authenticated
[cn3:21416] [[15336,0],2] tcp:peer_accept called for peer [[15336,1],4] in state UNKNOWN on socket 12
[cn3:21416] [[15336,0],2] SEND CONNECT ACK
[cn3:21416] [[15336,0],2] send blocking of 47 bytes to socket 12
[cn3:21416] [[15336,0],2] blocking send complete to socket 12
[cn3:21416] [[15336,0],2]-[[15336,1],4] tcp_peer_connected on socket 12
[cn3:21416] [[15336,0],2] routed_binomial_update: [[15336,1],4] --> [[15336,1],4]
[cn3:21416] [[15336,0],2]-[[15336,1],4] accepted: 192.168.1.3 - 192.168.1.3 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 743 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:set_module called for peer [[15336,1],4]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],4]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 62
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 826 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 831 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 104 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 267 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 738 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 202 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 443 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 190 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1313 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 164 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 160 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 289 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 206 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 104 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 713 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 665 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 684 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] connection_event_handler: working connection (13, 11) 192.168.1.3:40990
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] accept_connection: 192.168.1.3:40990
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 398 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 152 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1183 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:tcp:recv:handler called
[cn3:21416] [[15336,0],2] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 13
[cn3:21416] [[15336,0],2] waiting for connect ack from UNKNOWN
[cn3:21416] [[15336,0],2] connect ack received from UNKNOWN
[cn3:21416] [[15336,0],2] connect-ack recvd from UNKNOWN
[cn3:21416] [[15336,0],2] mca_oob_tcp_recv_connect: connection from new peer
[cn3:21416] [[15336,0],2] connect-ack header from [[15336,1],5] is okay
[cn3:21416] [[15336,0],2] waiting for connect ack from [[15336,1],5]
[cn3:21416] [[15336,0],2] connect ack received from [[15336,1],5]
[cn3:21416] [[15336,0],2] connect-ack version from [[15336,1],5] matches ours
[cn3:21416] [[15336,0],2] connect-ack [[15336,1],5] authenticated
[cn3:21416] [[15336,0],2] tcp:peer_accept called for peer [[15336,1],5] in state UNKNOWN on socket 13
[cn3:21416] [[15336,0],2] SEND CONNECT ACK
[cn3:21416] [[15336,0],2] send blocking of 47 bytes to socket 13
[cn3:21416] [[15336,0],2] blocking send complete to socket 13
[cn3:21416] [[15336,0],2]-[[15336,1],5] tcp_peer_connected on socket 13
[cn3:21416] [[15336,0],2] routed_binomial_update: [[15336,1],5] --> [[15336,1],5]
[cn3:21416] [[15336,0],2]-[[15336,1],5] accepted: 192.168.1.3 - 192.168.1.3 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 575 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:set_module called for peer [[15336,1],5]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 106 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 810 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 126 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 443 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1217 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],5]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 62
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1256 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 275 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 267 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 104 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 898 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 102 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 118 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 160 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 289 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 286 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1253 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 110 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 753 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 953 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] connection_event_handler: working connection (13, 0) 192.168.1.4:38801
[cn4:17828] [[15336,0],3] accept_connection: 192.168.1.4:38801
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 829 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 743 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:tcp:recv:handler called
[cn4:17828] [[15336,0],3] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 13
[cn4:17828] [[15336,0],3] waiting for connect ack from UNKNOWN
[cn4:17828] [[15336,0],3] connect ack received from UNKNOWN
[cn4:17828] [[15336,0],3] connect-ack recvd from UNKNOWN
[cn4:17828] [[15336,0],3] mca_oob_tcp_recv_connect: connection from new peer
[cn4:17828] [[15336,0],3] connect-ack header from [[15336,1],6] is okay
[cn4:17828] [[15336,0],3] waiting for connect ack from [[15336,1],6]
[cn4:17828] [[15336,0],3] connect ack received from [[15336,1],6]
[cn4:17828] [[15336,0],3] connect-ack version from [[15336,1],6] matches ours
[cn4:17828] [[15336,0],3] connect-ack [[15336,1],6] authenticated
[cn4:17828] [[15336,0],3] tcp:peer_accept called for peer [[15336,1],6] in state UNKNOWN on socket 13
[cn4:17828] [[15336,0],3] SEND CONNECT ACK
[cn4:17828] [[15336,0],3] send blocking of 47 bytes to socket 13
[cn4:17828] [[15336,0],3] blocking send complete to socket 13
[cn4:17828] [[15336,0],3]-[[15336,1],6] tcp_peer_connected on socket 13
[cn4:17828] [[15336,0],3] routed_binomial_update: [[15336,1],6] --> [[15336,1],6]
[cn4:17828] [[15336,0],3]-[[15336,1],6] accepted: 192.168.1.4 - 192.168.1.4 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:set_module called for peer [[15336,1],6]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 195 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1069 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],6]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 62
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 423 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 267 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 738 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 190 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 160 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 289 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 228 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 691 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 665 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1406 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] connection_event_handler: working connection (14, 11) 192.168.1.4:38802
[cn4:17828] [[15336,0],3] accept_connection: 192.168.1.4:38802
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 908 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:tcp:recv:handler called
[cn4:17828] [[15336,0],3] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 14
[cn4:17828] [[15336,0],3] waiting for connect ack from UNKNOWN
[cn4:17828] [[15336,0],3] connect ack received from UNKNOWN
[cn4:17828] [[15336,0],3] connect-ack recvd from UNKNOWN
[cn4:17828] [[15336,0],3] mca_oob_tcp_recv_connect: connection from new peer
[cn4:17828] [[15336,0],3] connect-ack header from [[15336,1],7] is okay
[cn4:17828] [[15336,0],3] waiting for connect ack from [[15336,1],7]
[cn4:17828] [[15336,0],3] connect ack received from [[15336,1],7]
[cn4:17828] [[15336,0],3] connect-ack version from [[15336,1],7] matches ours
[cn4:17828] [[15336,0],3] connect-ack [[15336,1],7] authenticated
[cn4:17828] [[15336,0],3] tcp:peer_accept called for peer [[15336,1],7] in state UNKNOWN on socket 14
[cn4:17828] [[15336,0],3] SEND CONNECT ACK
[cn4:17828] [[15336,0],3] send blocking of 47 bytes to socket 14
[cn4:17828] [[15336,0],3] blocking send complete to socket 14
[cn4:17828] [[15336,0],3]-[[15336,1],7] tcp_peer_connected on socket 14
[cn4:17828] [[15336,0],3] routed_binomial_update: [[15336,1],7] --> [[15336,1],7]
[cn4:17828] [[15336,0],3]-[[15336,1],7] accepted: 192.168.1.4 - 192.168.1.4 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 805 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:set_module called for peer [[15336,1],7]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 133 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1217 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],7]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 62
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 275 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],8]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],2]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,1],8] (ORIGIN [[15336,1],8]) OF 62 BYTES FOR DEST [[15336,0],4] TAG 1
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],6]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],4]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,1],4] (ORIGIN [[15336,1],4]) OF 62 BYTES FOR DEST [[15336,0],2] TAG 1
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] message received 62 bytes from [[15336,1],4] for tag 1
[cn3:21416] [[15336,0],2] orted:comm:process_commands() Processing Command: ORTE_DAEMON_SYNC_WANT_NIDMAP
[cn3:21416] [[15336,0],2] orted_recv: received sync+nidmap from local proc [[15336,1],4]
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,1],2] (ORIGIN [[15336,1],2]) OF 62 BYTES FOR DEST [[15336,0],1] TAG 1
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn5:14171] [[15336,0],4] message received 62 bytes from [[15336,1],8] for tag 1
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,1],6] (ORIGIN [[15336,1],6]) OF 62 BYTES FOR DEST [[15336,0],3] TAG 1
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,1],4] at tag 20
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,1],4] at tag 20
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,1],4]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,1],4]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,1],4]:20
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,1],4]
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] message received 62 bytes from [[15336,1],2] for tag 1
[cn2:01042] [[15336,0],1] orted:comm:process_commands() Processing Command: ORTE_DAEMON_SYNC_WANT_NIDMAP
[cn5:14171] [[15336,0],4] orted:comm:process_commands() Processing Command: ORTE_DAEMON_SYNC_WANT_NIDMAP
[cn5:14171] [[15336,0],4] orted_recv: received sync+nidmap from local proc [[15336,1],8]
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] message received 62 bytes from [[15336,1],6] for tag 1
[cn4:17828] [[15336,0],3] orted:comm:process_commands() Processing Command: ORTE_DAEMON_SYNC_WANT_NIDMAP
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],4]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],4]:20
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,1],4]) --> [[15336,1],4]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,1],4] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],4]
[cn2:01042] [[15336,0],1] orted_recv: received sync+nidmap from local proc [[15336,1],2]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,1],8] at tag 20
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,1],8] at tag 20
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,1],8]:20
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,1],8]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],8]
[cn4:17828] [[15336,0],3] orted_recv: received sync+nidmap from local proc [[15336,1],6]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,1],4]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,1],4]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,1],2] at tag 20
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],8]:20
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,1],8]) --> [[15336,1],8]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,1],8] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],8]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,1],6] at tag 20
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,1],4] OF 14140 BYTES ON SOCKET 12
[cn3:21416] [[15336,0],2]-[[15336,1],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,1],2] at tag 20
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,1],8]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,1],8]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,1],6] at tag 20
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,1],2]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,1],2]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,1],8] OF 14140 BYTES ON SOCKET 12
[cn5:14171] [[15336,0],4]-[[15336,1],8] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,1],6]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,1],2]:20
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,1],2]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,1],6]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,1],6]:20
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 273 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],2]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],2]:20
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,1],6]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],6]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,1],2]) --> [[15336,1],2]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,1],2] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],2]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],6]:20
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,1],2]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,1],2]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,1],6]) --> [[15336,1],6]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,1],6] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],6]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,1],2] OF 14140 BYTES ON SOCKET 13
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,1],6]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 108 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]-[[15336,1],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,1],6]
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],5]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,1],5] (ORIGIN [[15336,1],5]) OF 62 BYTES FOR DEST [[15336,0],2] TAG 1
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] message received 62 bytes from [[15336,1],5] for tag 1
[cn3:21416] [[15336,0],2] orted:comm:process_commands() Processing Command: ORTE_DAEMON_SYNC_WANT_NIDMAP
[cn3:21416] [[15336,0],2] orted_recv: received sync+nidmap from local proc [[15336,1],5]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,1],6] OF 14140 BYTES ON SOCKET 13
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,1],5] at tag 20
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,1],5] at tag 20
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,1],5]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,1],5]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,1],5]:20
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,1],5]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],5]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,1],6] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],5]:20
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,1],5]) --> [[15336,1],5]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,1],5] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],5]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 273 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,1],5]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,1],5]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,1],5] OF 14140 BYTES ON SOCKET 13
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,1],5] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] mca_oob_tcp_listen_thread: new connection: (20, 11) 192.168.1.1:35082
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 5
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 108 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 160 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],9]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,1],9] (ORIGIN [[15336,1],9]) OF 62 BYTES FOR DEST [[15336,0],4] TAG 1
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] message received 62 bytes from [[15336,1],9] for tag 1
[cn5:14171] [[15336,0],4] orted:comm:process_commands() Processing Command: ORTE_DAEMON_SYNC_WANT_NIDMAP
[cn5:14171] [[15336,0],4] orted_recv: received sync+nidmap from local proc [[15336,1],9]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,1],3] (ORIGIN [[15336,1],3]) OF 62 BYTES FOR DEST [[15336,0],1] TAG 1
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,1],9] at tag 20
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,1],9] at tag 20
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,1],9]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,1],9]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,1],9]:20
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] message received 62 bytes from [[15336,1],3] for tag 1
[cn2:01042] [[15336,0],1] orted:comm:process_commands() Processing Command: ORTE_DAEMON_SYNC_WANT_NIDMAP
[cn2:01042] [[15336,0],1] orted_recv: received sync+nidmap from local proc [[15336,1],3]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,1],9]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],9]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],9]:20
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,1],9]) --> [[15336,1],9]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 351 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,1],3] at tag 20
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,1],3] at tag 20
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,1],3]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,1],3]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,1],3]:20
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,1],3]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,1],9] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],9]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,1],9]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,1],9]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],3]:20
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,1],9] OF 14140 BYTES ON SOCKET 13
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],7]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,1],7] (ORIGIN [[15336,1],7]) OF 62 BYTES FOR DEST [[15336,0],3] TAG 1
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] message received 62 bytes from [[15336,1],7] for tag 1
[cn4:17828] [[15336,0],3] orted:comm:process_commands() Processing Command: ORTE_DAEMON_SYNC_WANT_NIDMAP
[cn4:17828] [[15336,0],3] orted_recv: received sync+nidmap from local proc [[15336,1],7]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,1],3]) --> [[15336,1],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,1],3] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],3]
[cn5:14171] [[15336,0],4]-[[15336,1],9] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,1],7] at tag 20
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,1],7] at tag 20
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,1],7]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,1],7]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,1],7]:20
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,1],7]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],7]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,1],3]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,1],3]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],7]:20
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,1],7]) --> [[15336,1],7]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,1],7] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],7]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,1],3] OF 14140 BYTES ON SOCKET 14
[cn2:01042] [[15336,0],1]-[[15336,1],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 5
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,1],7]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,1],7]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 5
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,1],7] OF 14140 BYTES ON SOCKET 14
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 273 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,1],7] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 5
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 108 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] connection_handler: working connection (20, 0) 192.168.1.1:35082
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] accept_connection: 192.168.1.1:35082
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 117
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 117 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 117
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 117 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 273 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 117
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 351 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 117 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 117
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 117 BYTES FOR DEST [[15336,0],0] TAG 5
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 117 bytes from [[15336,0],1] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 2 state RUNNING exit_code 0
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 3 state RUNNING exit_code 0
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn1:14209] [[15336,0],0] message received 117 bytes from [[15336,0],4] for tag 5
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],4]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 8 state RUNNING exit_code 0
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 9 state RUNNING exit_code 0
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 117 bytes from [[15336,0],3] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 6 state RUNNING exit_code 0
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 7 state RUNNING exit_code 0
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 108 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 117 bytes from [[15336,0],2] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 4 state RUNNING exit_code 0
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 5 state RUNNING exit_code 0
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 311
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 311 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 608 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 311
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 555 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 311 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 81
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 81 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 81
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 81 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 156 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called
[cn1:14209] [[15336,0],0] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 20
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] waiting for connect ack from UNKNOWN
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] connect ack received from UNKNOWN
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] connect-ack recvd from UNKNOWN
[cn1:14209] [[15336,0],0] mca_oob_tcp_recv_connect: connection from new peer
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 351 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] connect-ack header from [[15336,1],0] is okay
[cn1:14209] [[15336,0],0] waiting for connect ack from [[15336,1],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] connect ack received from [[15336,1],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] connect-ack version from [[15336,1],0] matches ours
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 83 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] connect-ack [[15336,1],0] authenticated
[cn1:14209] [[15336,0],0] tcp:peer_accept called for peer [[15336,1],0] in state UNKNOWN on socket 20
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] SEND CONNECT ACK
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] send blocking of 47 bytes to socket 20
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] blocking send complete to socket 20
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]-[[15336,1],0] tcp_peer_connected on socket 20
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] routed_binomial_update: [[15336,1],0] --> [[15336,1],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]-[[15336,1],0] accepted: 192.168.1.1 - 192.168.1.1 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14224] progress: initialized event flag to: 3
[cn1:14224] progress: initialized yield_when_idle to: false
[cn1:14224] progress: initialized num users to: 0
[cn1:14224] progress: initialized poll rate to: 23940000
[cn1:14224] progress: event_users_increment setting count to 1
[cn1:14224] mca: base: components_register: registering ess components
[cn1:14224] mca: base: components_register: found loaded component env
[cn1:14224] mca: base: components_register: component env has no register or open function
[cn1:14224] mca: base: components_open: opening ess components
[cn1:14224] mca: base: components_open: found loaded component env
[cn1:14224] mca: base: components_open: component env open function successful
[cn1:14224] mca:base:select: Auto-selecting ess components
[cn1:14224] mca:base:select:(  ess) Querying component [env]
[cn1:14224] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn1:14224] mca:base:select:(  ess) Selected component [env]
[cn1:14224] ess:env set name to [[15336,1],0]
[cn1:14224] mca: base: components_register: registering oob components
[cn1:14224] mca: base: components_register: found loaded component tcp
[cn1:14224] mca: base: components_register: component tcp register function successful
[cn1:14224] mca: base: components_open: opening oob components
[cn1:14224] mca: base: components_open: found loaded component tcp
[cn1:14224] mca: base: components_open: component tcp open function successful
[cn1:14224] mca:oob:select: checking available component tcp
[cn1:14224] mca:oob:select: Querying component [tcp]
[cn1:14224] oob:tcp: component_available called
[cn1:14224] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn1:14224] [[15336,1],0] oob:tcp:init rejecting loopback interface lo
[cn1:14224] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn1:14224] [[15336,1],0] oob:tcp:init creating module for V4 address on interface eth0
[cn1:14224] [[15336,1],0] creating OOB-TCP module for interface eth0
[cn1:14224] [[15336,1],0] oob:tcp:init adding 192.168.1.1 to our list of V4 connections
[cn1:14224] WORKING INTERFACE 3 KERNEL INDEX 3 FAMILY: V4
[cn1:14224] [[15336,1],0] oob:tcp:init creating module for V4 address on interface eth1
[cn1:14224] [[15336,1],0] creating OOB-TCP module for interface eth1
[cn1:14224] [[15336,1],0] oob:tcp:init adding 192.168.10.2 to our list of V4 connections
[cn1:14224] [[15336,1],0] TCP STARTUP
[cn1:14224] [[15336,1],0] attempting to bind to IPv4 port 0
[cn1:14224] [[15336,1],0] assigned IPv4 port 60242
[cn1:14224] mca:oob:select: Adding component to end
[cn1:14224] mca:oob:select: Found 1 active transports
[cn1:14224] mca: base: components_register: registering rml components
[cn1:14224] mca: base: components_register: found loaded component oob
[cn1:14224] mca: base: components_register: component oob has no register or open function
[cn1:14224] mca: base: components_open: opening rml components
[cn1:14224] mca: base: components_open: found loaded component oob
[cn1:14224] mca: base: components_open: component oob open function successful
[cn1:14224] orte_rml_base_select: initializing rml component oob
[cn1:14224] mca: base: components_register: registering routed components
[cn1:14224] mca: base: components_register: found loaded component binomial
[cn1:14224] mca: base: components_register: component binomial has no register or open function
[cn1:14224] mca: base: components_register: found loaded component debruijn
[cn1:14224] mca: base: components_register: component debruijn has no register or open function
[cn1:14224] mca: base: components_register: found loaded component direct
[cn1:14224] mca: base: components_register: component direct has no register or open function
[cn1:14224] mca: base: components_register: found loaded component radix
[cn1:14224] mca: base: components_register: component radix register function successful
[cn1:14224] mca: base: components_open: opening routed components
[cn1:14224] mca: base: components_open: found loaded component binomial
[cn1:14224] mca: base: components_open: found loaded component debruijn
[cn1:14224[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 311 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 156 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 311 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 81 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 81 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 838 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:set_module called for peer [[15336,1],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 779
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 779 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 260
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 482 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 260 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 90
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 90 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
] mca: base: components_open: found loaded component direct
[cn1:14224] mca: base: components_open: found loaded component radix
[cn1:14224] mca:base:select: Auto-selecting routed components
[cn1:14224] mca:base:select:(routed) Querying component [binomial]
[cn1:14224] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn1:14224] mca:base:select:(routed) Querying component [debruijn]
[cn1:14224] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn1:14224] mca:base:select:(routed) Querying component [direct]
[cn1:14224] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn1:14224] mca:base:select:(routed) Querying component [radix]
[cn1:14224] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn1:14224] mca:base:select:(routed) Selected component [binomial]
[cn1:14224] mca: base: close: unloading component debruijn
[cn1:14224] mca: base: close: unloading component direct
[cn1:14224] mca: base: close: unloading component radix
[cn1:14224] orte_routed_base_select: initializing selected component binomial
[cn1:14224] mca: base: components_register: registering dstore components
[cn1:14224] mca: base: components_register: found loaded component hash
[cn1:14224] mca: base: components_register: component hash register function successful
[cn1:14224] mca: base: components_open: opening dstore components
[cn1:14224] mca: base: components_open: found loaded component hash
[cn1:14224] mca:dstore:select: checking available component hash
[cn1:14224] mca:dstore:select: Querying component [hash]
[cn1:14224] mca: base: components_register: registering grpcomm components
[cn1:14224] mca: base: components_register: found loaded component bad
[cn1:14224] mca: base: components_register: component bad has no register or open function
[cn1:14224] mca: base: components_open: opening grpcomm components
[cn1:14224] mca: base: components_open: found loaded component bad
[cn1:14224] mca: base: components_open: component bad open function successful
[cn1:14224] mca:base:select: Auto-selecting grpcomm components
[cn1:14224] mca:base:select:(grpcomm) Querying component [bad]
[cn1:14224] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn1:14224] mca:base:select:(grpcomm) Selected component [bad]
[cn1:14224] [[15336,1],0] grpcomm:base:receive start comm
[cn1:14224] [[15336,1],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn1:14224] [[15336,1],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 48
[cn1:14224] [[15336,1],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn1:14224] [[15336,1],0] setting up session dir with
	tmpdir: /tmp
	host cn1
[cn1:14224] [[15336,1],0] posting recv
[cn1:14224] [[15336,1],0] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn1:14224] [[15336,1],0] posting recv
[cn1:14224] [[15336,1],0] posting persistent recv on tag 48 for peer [[WILDCARD],WILDCARD]
[cn1:14224] [[15336,1],0] posting recv
[cn1:14224] [[15336,1],0] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn1:14224] procdir: /tmp/openmpi-sessions-artpol@cn1_0/15336/1/0
[cn1:14224] jobdir: /tmp/openmpi-sessions-artpol@cn1_0/15336/1
[cn1:14224] top: openmpi-sessions-artpol@cn1_0
[cn1:14224] tmp: /tmp
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key opal.job.session.dir[OPAL_STRING] for proc 1005060097
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key opal.my.session.dir[OPAL_STRING] for proc 1005060097
[cn1:14224] [[15336,1],0] routed_binomial: init routes for proc job [15336,1]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
	daemon uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn1:14224] [[15336,1],0] [[15336,0],0] with daemon registering
[cn1:14224] [[15336,1],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 20
[cn1:14224] [[15336,1],0] rml_send_buffer to peer [[1[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01045] progress: initialized event flag to: 3
[cn2:01045] progress: initialized yield_when_idle to: false
[cn2:01045] progress: initialized num users to: 0
[cn2:01045] progress: initialized poll rate to: 23940000
[cn2:01045] progress: event_users_increment setting count to 1
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14174] progress: initialized event flag to: 3
[cn5:14174] progress: initialized yield_when_idle to: false
[cn5:14174] progress: initialized num users to: 0
[cn5:14174] progress: initialized poll rate to: 23940000
[cn5:14174] progress: event_users_increment setting count to 1
[cn4:17831] progress: initialized event flag to: 3
[cn3:21419] progress: initialized event flag to: 3
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 429 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 81
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 81 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 779 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 260 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 90 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 156 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 81 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 608 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
5336,0],0] at tag 1
[cn1:14224] [[15336,1],0] registering sync waiting for ack
[cn1:14224] [[15336,1],0] waiting on progress thread at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/routed/base/routed_base_fns.c:350
[cn1:14224] [[15336,1],0]: set_addr to uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn1:14224] [[15336,1],0]:set_addr checking if peer [[15336,0],0] is reachable via component tcp
[cn1:14224] [[15336,1],0] oob:tcp: working peer [[15336,0],0] address tcp://192.168.1.1,192.168.10.2:37347
[cn1:14224] [[15336,1],0] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn1:14224] [[15336,1],0] PASSING ADDR 192.168.1.1 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn1:14224] [[15336,1],0]:tcp set addr for peer [[15336,0],0]
[cn1:14224] [[15336,1],0] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 3 INTERFACE eth1
[cn1:14224] [[15336,1],0] PASSING ADDR 192.168.10.2 TO INTERFACE eth1 AT KERNEL INDEX 3
[cn1:14224] [[15336,1],0]:tcp set addr for peer [[15336,0],0]
[cn1:14224] [[15336,1],0]: peer [[15336,0],0] is reachable via component tcp
[cn1:14224] [[15336,1],0] posting recv
[cn1:14224] [[15336,1],0] posting non-persistent recv on tag 20 for peer [[WILDCARD],WILDCARD]
[cn1:14224] [[15336,1],0] rml_send_msg to peer [[15336,0],0] at tag 1
[cn1:14224] [[15336,1],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14224] [[15336,1],0]:tcp:processing set_peer cmd for interface eth0
[cn1:14224] [[15336,1],0] SET_PEER ADDING PEER [[15336,0],0]
[cn1:14224] [[15336,1],0] set_peer: peer [[15336,0],0] is listening on interface eth0 net 192.168.1.1 port 37347
[cn1:14224] [[15336,1],0]:tcp:processing set_peer cmd for interface eth1
[cn1:14224] [[15336,1],0] SET_PEER ADDING PEER [[15336,0],0]
[cn1:14224] [[15336,1],0] set_peer: peer [[15336,0],0] is listening on interface eth1 net 192.168.10.2 port 37347
[cn1:14224] [[15336,1],0] oob:base:send to target [[15336,0],0]
[cn1:14224] [[15336,1],0] oob:tcp:send_nb to peer [[15336,0],0]:1
[cn1:14224] [[15336,1],0] tcp:send_nb to peer [[15336,0],0]
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:1
[cn1:14224] [[15336,1],0] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_nb: initiating connection to [[15336,0],0]
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],0]
[cn1:14224] [[15336,1],0] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0
[cn1:14224] [[15336,1],0] oob:tcp:peer creating socket to [[15336,0],0]
[cn1:14224] [[15336,1],0] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0 on socket 11
[cn1:14224] [[15336,1],0] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] on 192.168.1.1:37347 - 0 retries
[cn1:14224] [[15336,1],0] waiting for connect completion to [[15336,0],0] - activating send event
[cn1:14224] [[15336,1],0] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_handler CONNECTING
[cn1:14224] [[15336,1],0]:tcp:complete_connect called for peer [[15336,0],0] on socket 11
[cn1:14224] [[15336,1],0] tcp_peer_complete_connect: sending ack to [[15336,0],0]
[cn1:14224] [[15336,1],0] SEND CONNECT ACK
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key opal.cred on proc 1005060097
[cn1:14224] dstore_hash:fetch key opal.cred for proc 1005060097 not found
[cn1:14224] [[15336,1],0] send blocking of 47 bytes to socket 11
[cn1:14224] [[15336,1],0] blocking send complete to socket 11
[cn1:14224] [[15336,1],0] tcp_peer_complete_[cn5:14174] mca: base: components_register: registering ess components
[cn5:14174] mca: base: components_register: found loaded component env
[cn5:14174] mca: base: components_register: component env has no register or open function
[cn5:14174] mca: base: components_open: opening ess components
[cn5:14174] mca: base: components_open: found loaded component env
[cn5:14174] mca: base: components_open: component env open function successful
[cn5:14174] mca:base:select: Auto-selecting ess components
[cn5:14174] mca:base:select:(  ess) Querying component [env]
[cn5:14174] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn5:14174] mca:base:select:(  ess) Selected component [env]
[cn5:14174] ess:env set name to [[15336,1],8]
[cn4:17831] progress: initialized yield_when_idle to: false
[cn4:17831] progress: initialized num users to: 0
[cn4:17831] progress: initialized poll rate to: 23940000
[cn4:17831] progress: event_users_increment setting count to 1
[cn3:21419] progress: initialized yield_when_idle to: false
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn2:01046] progress: initialized event flag to: 3
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 260
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 260 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 311
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 311 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 311
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 311 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 260 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2905 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 311 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 83 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 311 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 429 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
connect: setting read event on connection to [[15336,0],0]
[cn2:01046] progress: initialized yield_when_idle to: false
[cn2:01046] progress: initialized num users to: 0
[cn2:01046] progress: initialized poll rate to: 23940000
[cn2:01046] progress: event_users_increment setting count to 1
[cn5:14175] progress: initialized event flag to: 3
[cn5:14175] progress: initialized yield_when_idle to: false
[cn5:14175] progress: initialized num users to: 0
[cn5:14175] progress: initialized poll rate to: 23940000
[cn5:14175] progress: event_users_increment setting count to 1
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17832] progress: initialized event flag to: 3
[cn4:17832] progress: initialized yield_when_idle to: false
[cn4:17832] progress: initialized num users to: 0
[cn4:17832] progress: initialized poll rate to: 23940000
[cn4:17832] progress: event_users_increment setting count to 1
[cn3:21419] progress: initialized num users to: 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 263
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 263 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 779
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 779 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 263
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 263 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 87
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 87 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 263 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 779 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 263 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 555 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 87 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01045] mca: base: components_register: registering ess components
[cn2:01045] mca: base: components_register: found loaded component env
[cn2:01045] mca: base: components_register: component env has no register or open function
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14175] mca: base: components_register: registering ess components
[cn5:14175] mca: base: components_register: found loaded component env
[cn5:14175] mca: base: components_register: component env has no register or open function
[cn5:14175] mca: base: components_open: opening ess components
[cn5:14175] mca: base: components_open: found loaded component env
[cn5:14175] mca: base: components_open: component env open function successful
[cn5:14175] mca:base:select: Auto-selecting ess components
[cn5:14175] mca:base:select:(  ess) Querying component [env]
[cn5:14175] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn5:14175] mca:base:select:(  ess) Selected component [env]
[cn5:14175] ess:env set name to [[15336,1],9]
[cn4:17831] mca: base: components_register: registering ess components
[cn4:17831] mca: base: components_register: found loaded component env
[cn4:17831] mca: base: components_register: component env has no register or open function
[cn3:21419] progress: initialized poll rate to: 23940000
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2221 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 500
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 83 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 500 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 172
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 172 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 546
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 546 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 83 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 500 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 172 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 546 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01045] mca: base: components_open: opening ess components
[cn2:01045] mca: base: components_open: found loaded component env
[cn2:01045] mca: base: components_open: component env open function successful
[cn2:01045] mca:base:select: Auto-selecting ess components
[cn2:01045] mca:base:select:(  ess) Querying component [env]
[cn2:01045] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn2:01045] mca:base:select:(  ess) Selected component [env]
[cn5:14174] mca: base: components_register: registering oob components
[cn5:14174] mca: base: components_register: found loaded component tcp
[cn4:17831] mca: base: components_open: opening ess components
[cn4:17831] mca: base: components_open: found loaded component env
[cn4:17831] mca: base: components_open: component env open function successful
[cn4:17831] mca:base:select: Auto-selecting ess components
[cn4:17831] mca:base:select:(  ess) Querying component [env]
[cn4:17831] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn4:17831] mca:base:select:(  ess) Selected component [env]
[cn4:17831] ess:env set name to [[15336,1],6]
[cn3:21419] progress: event_users_increment setting count to 1
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 76
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 76 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 117
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 117 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 882 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 733
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 733 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 311
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 311 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 76 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 117 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 733 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2771 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 311 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1601 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01045] ess:env set name to [[15336,1],2]
[cn5:14174] mca: base: components_register: component tcp register function successful
[cn4:17832] mca: base: components_register: registering ess components
[cn4:17832] mca: base: components_register: found loaded component env
[cn4:17832] mca: base: components_register: component env has no register or open function
[cn4:17832] mca: base: components_open: opening ess components
[cn4:17832] mca: base: components_open: found loaded component env
[cn4:17832] mca: base: components_open: component env open function successful
[cn4:17832] mca:base:select: Auto-selecting ess components
[cn4:17832] mca:base:select:(  ess) Querying component [env]
[cn4:17832] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn4:17832] mca:base:select:(  ess) Selected component [env]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21420] progress: initialized event flag to: 3
[cn3:21420] progress: initialized yield_when_idle to: false
[cn3:21420] progress: initialized num users to: 0
[cn3:21420] progress: initialized poll rate to: 23940000
[cn3:21420] progress: event_users_increment setting count to 1
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 882 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 401
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 401 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 76
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 76 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 401 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 76 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 192
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 192 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01046] mca: base: components_register: registering ess components
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14174] mca: base: components_open: opening oob components
[cn5:14174] mca: base: components_open: found loaded component tcp
[cn5:14174] mca: base: components_open: component tcp open function successful
[cn5:14174] mca:oob:select: checking available component tcp
[cn5:14174] mca:oob:select: Querying component [tcp]
[cn5:14174] oob:tcp: component_available called
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17832] ess:env set name to [[15336,1],7]
[cn3:21419] mca: base: components_register: registering ess components
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 192
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 192 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 462
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 462 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2748 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2076 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 192 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1893 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 192 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 462 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21419] mca: base: components_register: found loaded component env
[cn3:21419] mca: base: components_register: component env has no register or open function
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01046] mca: base: components_register: found loaded component env
[cn2:01046] mca: base: components_register: component env has no register or open function
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14174] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn5:14174] [[15336,1],8] oob:tcp:init rejecting loopback interface lo
[cn5:14174] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn5:14174] [[15336,1],8] oob:tcp:init creating module for V4 address on interface eth0
[cn5:14174] [[15336,1],8] creating OOB-TCP module for interface eth0
[cn5:14174] [[15336,1],8] oob:tcp:init adding 192.168.1.5 to our list of V4 connections
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17832] mca: base: components_register: registering oob components
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 546
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 546 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 227 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 231
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1447 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 231 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 172
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3403 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 172 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 546
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 546 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 546 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3571 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 231 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 172 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 546 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1643 BYTES ON SOCKET 9
[cn2:01046] mca: base: components_open: opening ess components
[cn2:01046] mca: base: components_open: found loaded component env
[cn2:01046] mca: base: components_open: component env open function successful
[cn2:01046] mca:base:select: Auto-selecting ess components
[cn2:01046] mca:base:select:(  ess) Querying component [env]
[cn2:01046] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn2:01046] mca:base:select:(  ess) Selected component [env]
[cn2:01046] ess:env set name to [[15336,1],3]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14174] [[15336,1],8] TCP STARTUP
[cn5:14174] [[15336,1],8] attempting to bind to IPv4 port 0
[cn5:14174] [[15336,1],8] assigned IPv4 port 59276
[cn5:14174] mca:oob:select: Adding component to end
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17831] mca: base: components_register: registering oob components
[cn4:17831] mca: base: components_register: found loaded component tcp
[cn3:21419] mca: base: components_open: opening ess components
[cn3:21419] mca: base: components_open: found loaded component env
[cn3:21419] mca: base: components_open: component env open function successful
[cn3:21419] mca:base:select: Auto-selecting ess components
[cn3:21419] mca:base:select:(  ess) Querying component [env]
[cn3:21419] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn3:21419] mca:base:select:(  ess) Selected component [env]
[cn3:21419] ess:env set name to [[15336,1],4]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1317
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1317 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 84
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2509 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 84 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3280 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 117
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 117 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1317 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1772 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 84 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 117 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 537
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 537 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1806 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3108 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 537
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 537 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 401
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 401 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 149 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 192
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 192 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2352 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn2:01045] mca: base: components_register: registering oob components
[cn2:01045] mca: base: components_register: found loaded component tcp
[cn2:01045] mca: base: components_register: component tcp register function successful
[cn2:01045] mca: base: components_open: opening oob components
[cn2:01045] mca: base: components_open: found loaded component tcp
[cn2:01045] mca: base: components_open: component tcp open function successful
[cn2:01045] mca:oob:select: checking available component tcp
[cn2:01045] mca:oob:select: Querying component [tcp]
[cn2:01045] oob:tcp: component_available called
[cn2:01045] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn2:01045] [[15336,1],2] oob:tcp:init rejecting loopback interface lo
[cn2:01045] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn2:01045] [[15336,1],2] oob:tcp:init creating module for V4 address on interface eth0
[cn2:01045] [[15336,1],2] creating OOB-TCP module for interface eth0
[cn2:01045] [[15336,1],2] oob:tcp:init adding 192.168.1.2 to our list of V4 connections
[cn2:01045] [[15336,1],2] TCP STARTUP
[cn2:01045] [[15336,1],2] attempting to bind to IPv4 port 0
[cn2:01045] [[15336,1],2] assigned IPv4 port 39416
[cn2:01045] mca:oob:select: Adding component to end
[cn2:01045] mca:oob:select: Found 1 active transports
[cn5:14174] mca:oob:select: Found 1 active transports
[cn4:17831] mca: base: components_register: component tcp register function successful
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21420] mca: base: components_register: registering ess components
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 537 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 537 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 401 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 192 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 387 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 276
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 276 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2396 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 217
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 217 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 546
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2445 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 546 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn2:01045] mca: base: components_register: registering rml components
[cn2:01045] mca: base: components_register: found loaded component oob
[cn2:01045] mca: base: components_register: component oob has no register or open function
[cn2:01045] mca: base: components_open: opening rml components
[cn2:01045] mca: base: components_open: found loaded component oob
[cn2:01045] mca: base: components_open: component oob open function successful
[cn2:01045] orte_rml_base_select: initializing rml component oob
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14174] mca: base: components_register: registering rml components
[cn5:14174] mca: base: components_register: found loaded component oob
[cn5:14174] mca: base: components_register: component oob has no register or open function
[cn5:14174] mca: base: components_open: opening rml components
[cn5:14174] mca: base: components_open: found loaded component oob
[cn5:14174] mca: base: components_open: component oob open function successful
[cn5:14174] orte_rml_base_select: initializing rml component oob
[cn4:17831] mca: base: components_open: opening oob components
[cn4:17831] mca: base: components_open: found loaded component tcp
[cn4:17831] mca: base: components_open: component tcp open function successful
[cn4:17831] mca:oob:select: checking available component tcp
[cn4:17831] mca:oob:select: Querying component [tcp]
[cn4:17831] oob:tcp: component_available called
[cn3:21420] mca: base: components_register: found loaded component env
[cn3:21420] mca: base: components_register: component env has no register or open function
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 276 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 217 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 546 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1898 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 608 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 443
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1745 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 443 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2289 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 424
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 424 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 172
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 172 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01046] mca: base: components_register: registering oob components
[cn5:14174] mca: base: components_register: registering routed components
[cn5:14174] mca: base: components_register: found loaded component binomial
[cn5:14174] mca: base: components_register: component binomial has no register or open function
[cn4:17831] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn4:17831] [[15336,1],6] oob:tcp:init rejecting loopback interface lo
[cn4:17831] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21420] mca: base: components_open: opening ess components
[cn3:21420] mca: base: components_open: found loaded component env
[cn3:21420] mca: base: components_open: component env open function successful
[cn3:21420] mca:base:select: Auto-selecting ess components
[cn3:21420] mca:base:select:(  ess) Querying component [env]
[cn3:21420] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn3:21420] mca:base:select:(  ess) Selected component [env]
[cn3:21420] ess:env set name to [[15336,1],5]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 443 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 424 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 172 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01046] mca: base: components_register: found loaded component tcp
[cn5:14174] mca: base: components_register: found loaded component debruijn
[cn5:14174] mca: base: components_register: component debruijn has no register or open function
[cn5:14174] mca: base: components_register: found loaded component direct
[cn5:14174] mca: base: components_register: component direct has no register or open function
[cn5:14174] mca: base: components_register: found loaded component radix
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17831] [[15336,1],6] oob:tcp:init creating module for V4 address on interface eth0
[cn4:17831] [[15336,1],6] creating OOB-TCP module for interface eth0
[cn4:17831] [[15336,1],6] oob:tcp:init adding 192.168.1.4 to our list of V4 connections
[cn4:17831] [[15336,1],6] TCP STARTUP
[cn4:17831] [[15336,1],6] attempting to bind to IPv4 port 0
[cn4:17831] [[15336,1],6] assigned IPv4 port 49793
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21419] mca: base: components_register: registering oob components
[cn3:21419] mca: base: components_register: found loaded component tcp
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 464 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1848 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 117
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 117 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1256
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1256 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 464 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2943 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 136
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 136 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 117
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 117 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 117 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1256 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 136 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 117 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01046] mca: base: components_register: component tcp register function successful
[cn5:14174] mca: base: components_register: component radix register function successful
[cn5:14174] mca: base: components_open: opening routed components
[cn5:14174] mca: base: components_open: found loaded component binomial
[cn5:14174] mca: base: components_open: found loaded component debruijn
[cn5:14174] mca: base: components_open: found loaded component direct
[cn5:14174] mca: base: components_open: found loaded component radix
[cn5:14174] mca:base:select: Auto-selecting routed components
[cn5:14174] mca:base:select:(routed) Querying component [binomial]
[cn5:14174] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn5:14174] mca:base:select:(routed) Querying component [debruijn]
[cn5:14174] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn5:14174] mca:base:select:(routed) Querying component [direct]
[cn5:14174] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn5:14174] mca:base:select:(routed) Querying component [radix]
[cn5:14174] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn5:14174] mca:base:select:(routed) Selected component [binomial]
[cn5:14174] mca: base: close: unloading component debruijn
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17831] mca:oob:select: Adding component to end
[cn4:17831] mca:oob:select: Found 1 active transports
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21419] mca: base: components_register: component tcp register function successful
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 401
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 401 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1844 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 788 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 303 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 401
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 401 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 464 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 788 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 401 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 401 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 462
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2324 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 462 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01046] mca: base: components_open: opening oob components
[cn2:01046] mca: base: components_open: found loaded component tcp
[cn2:01046] mca: base: components_open: component tcp open function successful
[cn2:01046] mca:oob:select: checking available component tcp
[cn2:01046] mca:oob:select: Querying component [tcp]
[cn2:01046] oob:tcp: component_available called
[cn5:14174] mca: base: close: unloading component direct
[cn5:14174] mca: base: close: unloading component radix
[cn5:14174] orte_routed_base_select: initializing selected component binomial
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17832] mca: base: components_register: found loaded component tcp
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21419] mca: base: components_open: opening oob components
[cn3:21419] mca: base: components_open: found loaded component tcp
[cn3:21419] mca: base: components_open: component tcp open function successful
[cn3:21419] mca:oob:select: checking available component tcp
[cn3:21419] mca:oob:select: Querying component [tcp]
[cn3:21419] oob:tcp: component_available called
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 717
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 233 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 717 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 520
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 520 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 117
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 117 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 462 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 717 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 520 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 117 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 238 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 172
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 172 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 855 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 401
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 401 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1242 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 106 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 855 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 285
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 285 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21419] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn3:21419] [[15336,1],4] oob:tcp:init rejecting loopback interface lo
[cn3:21419] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn3:21419] [[15336,1],4] oob:tcp:init creating module for V4 address on interface eth0
[cn3:21419] [[15336,1],4] creating OOB-TCP module for interface eth0
[cn3:21419] [[15336,1],4] oob:tcp:init adding 192.168.1.3 to our list of V4 connections
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01046] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn2:01046] [[15336,1],3] oob:tcp:init rejecting loopback interface lo
[cn2:01046] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn2:01046] [[15336,1],3] oob:tcp:init creating module for V4 address on interface eth0
[cn2:01046] [[15336,1],3] creating OOB-TCP module for interface eth0
[cn2:01046] [[15336,1],3] oob:tcp:init adding 192.168.1.2 to our list of V4 connections
[cn2:01046] [[15336,1],3] TCP STARTUP
[cn2:01046] [[15336,1],3] attempting to bind to IPv4 port 0
[cn2:01046] [[15336,1],3] assigned IPv4 port 37264
[cn2:01046] mca:oob:select: Adding component to end
[cn2:01046] mca:oob:select: Found 1 active transports
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn5:14174] mca: base: components_register: registering dstore components
[cn5:14174] mca: base: components_register: found loaded component hash
[cn5:14174] mca: base: components_register: component hash register function successful
[cn5:14174] mca: base: components_open: opening dstore components
[cn5:14174] mca: base: components_open: found loaded component hash
[cn5:14174] mca:dstore:select: checking available component hash
[cn5:14174] mca:dstore:select: Querying component [hash]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn4:17832] mca: base: components_register: component tcp register function successful
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 172 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 401 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 285 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01046] mca: base: components_register: registering rml components
[cn5:14175] mca: base: components_register: registering oob components
[cn5:14175] mca: base: components_register: found loaded component tcp
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17832] mca: base: components_open: opening oob components
[cn4:17832] mca: base: components_open: found loaded component tcp
[cn4:17832] mca: base: components_open: component tcp open function successful
[cn4:17832] mca:oob:select: checking available component tcp
[cn4:17832] mca:oob:select: Querying component [tcp]
[cn4:17832] oob:tcp: component_available called
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21419] [[15336,1],4] TCP STARTUP
[cn3:21419] [[15336,1],4] attempting to bind to IPv4 port 0
[cn3:21419] [[15336,1],4] assigned IPv4 port 50297
[cn3:21419] mca:oob:select: Adding component to end
[cn3:21419] mca:oob:select: Found 1 active transports
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 192
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 192 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1672 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1135
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1135 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 430 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1898 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 88
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 88 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 263
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 263 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 192 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 319 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 1135 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 88 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 263 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01046] mca: base: components_register: found loaded component oob
[cn2:01046] mca: base: components_register: component oob has no register or open function
[cn5:14174] mca: base: components_register: registering grpcomm components
[cn5:14174] mca: base: components_register: found loaded component bad
[cn5:14174] mca: base: components_register: component bad has no register or open function
[cn5:14174] mca: base: components_open: opening grpcomm components
[cn5:14174] mca: base: components_open: found loaded component bad
[cn5:14174] mca: base: components_open: component bad open function successful
[cn5:14174] mca:base:select: Auto-selecting grpcomm components
[cn5:14174] mca:base:select:(grpcomm) Querying component [bad]
[cn5:14174] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn5:14174] mca:base:select:(grpcomm) Selected component [bad]
[cn5:14174] [[15336,1],8] grpcomm:base:receive start comm
[cn5:14174] [[15336,1],8] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn5:14174] [[15336,1],8] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 48
[cn5:14174] [[15336,1],8] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn5:14174] [[15336,1],8] setting up session dir with
	tmpdir: /tmp
	host cn5
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17832] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21419] mca: base: components_register: registering rml components
[cn3:21419] mca: base: components_register: found loaded component oob
[cn3:21419] mca: base: components_register: component oob has no register or open function
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 319 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 304
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 304 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 464 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 488
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3050 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 464 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 488 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 247
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 171 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 247 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 304
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 304 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 304 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 488 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1648 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 247 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 304 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 276
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 276 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 323 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 788 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 374
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 374 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 788 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 442
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 442 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 172
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2968 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 172 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01046] mca: base: components_open: opening rml components
[cn2:01046] mca: base: components_open: found loaded component oob
[cn2:01046] mca: base: components_open: component oob open function successful
[cn2:01046] orte_rml_base_select: initializing rml component oob
[cn5:14175] mca: base: components_register: component tcp register function successful
[cn5:14175] mca: base: components_open: opening oob components
[cn5:14175] mca: base: components_open: found loaded component tcp
[cn5:14175] mca: base: components_open: component tcp open function successful
[cn5:14175] mca:oob:select: checking available component tcp
[cn5:14175] mca:oob:select: Querying component [tcp]
[cn5:14175] oob:tcp: component_available called
[cn4:17832] [[15336,1],7] oob:tcp:init rejecting loopback interface lo
[cn4:17832] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn4:17832] [[15336,1],7] oob:tcp:init creating module for V4 address on interface eth0
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21419] mca: base: components_open: opening rml components
[cn3:21419] mca: base: components_open: found loaded component oob
[cn3:21419] mca: base: components_open: component oob open function successful
[cn3:21419] orte_rml_base_select: initializing rml component oob
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 276 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 374 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 442 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 172 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 788 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01045] mca: base: components_register: registering routed components
[cn2:01045] mca: base: components_register: found loaded component binomial
[cn2:01045] mca: base: components_register: component binomial has no register or open function
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14175] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn5:14175] [[15336,1],9] oob:tcp:init rejecting loopback interface lo
[cn5:14175] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn5:14175] [[15336,1],9] oob:tcp:init creating module for V4 address on interface eth0
[cn5:14175] [[15336,1],9] creating OOB-TCP module for interface eth0
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17832] [[15336,1],7] creating OOB-TCP module for interface eth0
[cn4:17832] [[15336,1],7] oob:tcp:init adding 192.168.1.4 to our list of V4 connections
[cn4:17832] [[15336,1],7] TCP STARTUP
[cn4:17832] [[15336,1],7] attempting to bind to IPv4 port 0
[cn4:17832] [[15336,1],7] assigned IPv4 port 41093
[cn4:17832] mca:oob:select: Adding component to end
[cn4:17832] mca:oob:select: Found 1 active transports
[cn3:21420] mca: base: components_register: registering oob components
[cn3:21420] mca: base: components_register: found loaded component tcp
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 532
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 532 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 216
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1672 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 216 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 263
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 855 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 263 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 855 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 117
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 117 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 532 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 175 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 216 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 263 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 117 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3036 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 319 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 319 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1280
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1280 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 187
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 187 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 643 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 304
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 304 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2679 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 401
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 401 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01045] mca: base: components_register: found loaded component debruijn
[cn2:01045] mca: base: components_register: component debruijn has no register or open function
[cn2:01045] mca: base: components_register: found loaded component direct
[cn2:01045] mca: base: components_register: component direct has no register or open function
[cn2:01045] mca: base: components_register: found loaded component radix
[cn2:01045] mca: base: components_register: component radix register function successful
[cn5:14175] [[15336,1],9] oob:tcp:init adding 192.168.1.5 to our list of V4 connections
[cn5:14175] [[15336,1],9] TCP STARTUP
[cn5:14175] [[15336,1],9] attempting to bind to IPv4 port 0
[cn4:17831] mca: base: components_register: registering rml components
[cn4:17831] mca: base: components_register: found loaded component oob
[cn4:17831] mca: base: components_register: component oob has no register or open function
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21420] mca: base: components_register: component tcp register function successful
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1280 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 187 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 170 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 304 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 401 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01045] mca: base: components_open: opening routed components
[cn2:01045] mca: base: components_open: found loaded component binomial
[cn2:01045] mca: base: components_open: found loaded component debruijn
[cn2:01045] mca: base: components_open: found loaded component direct
[cn2:01045] mca: base: components_open: found loaded component radix
[cn2:01045] mca:base:select: Auto-selecting routed components
[cn2:01045] mca:base:select:(routed) Querying component [binomial]
[cn2:01045] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn2:01045] mca:base:select:(routed) Querying component [debruijn]
[cn2:01045] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn2:01045] mca:base:select:(routed) Querying component [direct]
[cn2:01045] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn2:01045] mca:base:select:(routed) Querying component [radix]
[cn2:01045] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn2:01045] mca:base:select:(routed) Selected component [binomial]
[cn2:01045] mca: base: close: unloading component debruijn
[cn2:01045] mca: base: close: unloading component direct
[cn2:01045] mca: base: close: unloading component radix
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14175] [[15336,1],9] assigned IPv4 port 47378
[cn5:14175] mca:oob:select: Adding component to end
[cn5:14175] mca:oob:select: Found 1 active transports
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17831] mca: base: components_open: opening rml components
[cn4:17831] mca: base: components_open: found loaded component oob
[cn4:17831] mca: base: components_open: component oob open function successful
[cn4:17831] orte_rml_base_select: initializing rml component oob
[cn3:21420] mca: base: components_open: opening oob components
[cn3:21420] mca: base: components_open: found loaded component tcp
[cn3:21420] mca: base: components_open: component tcp open function successful
[cn3:21420] mca:oob:select: checking available component tcp
[cn3:21420] mca:oob:select: Querying component [tcp]
[cn3:21420] oob:tcp: component_available called
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 108
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 108 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1253
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 343 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1253 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 88
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 88 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 108 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 193 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 117 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 1253 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 88 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 401
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 401 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 247
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 247 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01045] orte_routed_base_select: initializing selected component binomial
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14174] procdir: /tmp/openmpi-sessions-artpol@cn5_0/15336/1/8
[cn5:14174] jobdir: /tmp/openmpi-sessions-artpol@cn5_0/15336/1
[cn5:14174] top: openmpi-sessions-artpol@cn5_0
[cn5:14174] tmp: /tmp
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key opal.job.session.dir[OPAL_STRING] for proc 35364798465
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key opal.my.session.dir[OPAL_STRING] for proc 35364798465
[cn5:14174] [[15336,1],8] routed_binomial: init routes for proc job [15336,1]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
	daemon uri 1005060096.4;tcp://192.168.1.5:34732
[cn5:14174] [[15336,1],8] [[15336,0],4] with daemon registering
[cn5:14174] [[15336,1],8] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 20
[cn5:14174] [[15336,1],8] rml_send_buffer to peer [[15336,0],4] at tag 1
[cn5:14174] [[15336,1],8] registering sync waiting for ack
[cn5:14174] [[15336,1],8] waiting on progress thread at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/routed/base/routed_base_fns.c:350
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17832] mca: base: components_register: registering rml components
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21420] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 520
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 520 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 464 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 537
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 537 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 401 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 464 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 247 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 520 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 245 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 537 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17832] mca: base: components_register: found loaded component oob
[cn4:17832] mca: base: components_register: component oob has no register or open function
[cn4:17832] mca: base: components_open: opening rml components
[cn4:17832] mca: base: components_open: found loaded component oob
[cn4:17832] mca: base: components_open: component oob open function successful
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21420] [[15336,1],5] oob:tcp:init rejecting loopback interface lo
[cn3:21420] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn3:21420] [[15336,1],5] oob:tcp:init creating module for V4 address on interface eth0
[cn2:01045] mca: base: components_register: registering dstore components
[cn2:01045] mca: base: components_register: found loaded component hash
[cn2:01045] mca: base: components_register: component hash register function successful
[cn2:01045] mca: base: components_open: opening dstore components
[cn2:01045] mca: base: components_open: found loaded component hash
[cn2:01045] mca:dstore:select: checking available component hash
[cn2:01045] mca:dstore:select: Querying component [hash]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14175] mca: base: components_register: registering rml components
[cn5:14175] mca: base: components_register: found loaded component oob
[cn5:14175] mca: base: components_register: component oob has no register or open function
[cn5:14175] mca: base: components_open: opening rml components
[cn5:14175] mca: base: components_open: found loaded component oob
[cn5:14175] mca: base: components_open: component oob open function successful
[cn5:14175] orte_rml_base_select: initializing rml component oob
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 105
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 105 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 499
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 499 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 95
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 95 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 855 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 788 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 388
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 388 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 319 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 105 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 788 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 499 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 95 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 388 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 104
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 104 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 319 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 84
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 84 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01045] mca: base: components_register: registering grpcomm components
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14174] [[15336,1],8] posting recv
[cn5:14174] [[15336,1],8] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn5:14174] [[15336,1],8] posting recv
[cn5:14174] [[15336,1],8] posting persistent recv on tag 48 for peer [[WILDCARD],WILDCARD]
[cn5:14174] [[15336,1],8] posting recv
[cn5:14174] [[15336,1],8] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn5:14174] [[15336,1],8]: set_addr to uri 1005060096.4;tcp://192.168.1.5:34732
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17832] orte_rml_base_select: initializing rml component oob
[cn3:21420] [[15336,1],5] creating OOB-TCP module for interface eth0
[cn3:21420] [[15336,1],5] oob:tcp:init adding 192.168.1.3 to our list of V4 connections
[cn3:21420] [[15336,1],5] TCP STARTUP
[cn3:21420] [[15336,1],5] attempting to bind to IPv4 port 0
[cn3:21420] [[15336,1],5] assigned IPv4 port 37735
[cn3:21420] mca:oob:select: Adding component to end
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 855 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1060
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 855 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1060 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 948
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 948 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 104 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 84 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 319 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 1060 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 319 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 948 bytes from [[15336,0],4] for tag 2
[cn4:17831] mca: base: components_register: registering routed components
[cn3:21420] mca:oob:select: Found 1 active transports
[cn2:01045] mca: base: components_register: found loaded component bad
[cn2:01045] mca: base: components_register: component bad has no register or open function
[cn2:01045] mca: base: components_open: opening grpcomm components
[cn2:01045] mca: base: components_open: found loaded component bad
[cn2:01045] mca: base: components_open: component bad open function successful
[cn2:01045] mca:base:select: Auto-selecting grpcomm components
[cn2:01045] mca:base:select:(grpcomm) Querying component [bad]
[cn2:01045] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn2:01045] mca:base:select:(grpcomm) Selected component [bad]
[cn2:01045] [[15336,1],2] grpcomm:base:receive start comm
[cn2:01045] [[15336,1],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn2:01045] [[15336,1],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 48
[cn2:01045] [[15336,1],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn2:01045] [[15336,1],2] setting up session dir with
	tmpdir: /tmp
	host cn2
[cn5:14174] [[15336,1],8]:set_addr checking if peer [[15336,0],4] is reachable via component tcp
[cn5:14174] [[15336,1],8] oob:tcp: working peer [[15336,0],4] address tcp://192.168.1.5:34732
[cn5:14174] [[15336,1],8] PEER [[15336,0],4] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn5:14174] [[15336,1],8] PASSING ADDR 192.168.1.5 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn5:14174] [[15336,1],8]:tcp set addr for peer [[15336,0],4]
[cn5:14174] [[15336,1],8]: peer [[15336,0],4] is reachable via component tcp
[cn5:14174] [[15336,1],8] posting recv
[cn5:14174] [[15336,1],8] posting non-persistent recv on tag 20 for peer [[WILDCARD],WILDCARD]
[cn5:14174] [[15336,1],8] rml_send_msg to peer [[15336,0],4] at tag 1
[cn5:14174] [[15336,1],8] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14174] [[15336,1],8]:tcp:processing set_peer cmd for interface eth0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2106
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2106 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1110
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1110 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 202
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 202 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 263
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 263 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 2106 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 1110 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 202 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 263 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 75
[cn2:01046] mca: base: components_register: registering routed components
[cn2:01046] mca: base: components_register: found loaded component binomial
[cn2:01046] mca: base: components_register: component binomial has no register or open function
[cn2:01046] mca: base: components_register: found loaded component debruijn
[cn2:01046] mca: base: components_register: component debruijn has no register or open function
[cn2:01046] mca: base: components_register: found loaded component direct
[cn2:01046] mca: base: components_register: component direct has no register or open function
[cn2:01046] mca: base: components_register: found loaded component radix
[cn2:01046] mca: base: components_register: component radix register function successful
[cn2:01046] mca: base: components_open: opening routed components
[cn2:01046] mca: base: components_open: found loaded component binomial
[cn2:01046] mca: base: components_open: found loaded component debruijn
[cn2:01046] mca: base: components_open: found loaded component direct
[cn2:01046] mca: base: components_open: found loaded component radix
[cn2:01046] mca:base:select: Auto-selecting routed components
[cn2:01046] mca:base:select:(routed) Querying component [binomial]
[cn2:01046] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn2:01046] mca:base:select:(routed) Querying component [debruijn]
[cn2:01046] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn2:01046] mca:base:select:(routed) Querying component [direct]
[cn2:01046] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn2:01046] mca:base:select:(routed) Querying component [radix]
[cn2:01046] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn2:01046] mca:base:select:(routed) Selected component [binomial]
[cn2:01046] mca: base: close: unloading component debruijn
[cn2:01046] mca: base: close: unloading component direct
[cn2:01046] mca: base: close: unloading component radix
[cn2:01046] orte_routed_base_select: initializing selected component binomial
[cn5:14174] [[15336,1],8] SET_PEER ADDING PEER [[15336,0],4]
[cn5:14174] [[15336,1],8] set_peer: peer [[15336,0],4] is listening on interface eth0 net 192.168.1.5 port 34732
[cn5:14174] [[15336,1],8] oob:base:send to target [[15336,0],4]
[cn5:14174] [[15336,1],8] oob:tcp:send_nb to peer [[15336,0],4]:1
[cn5:14174] [[15336,1],8] tcp:send_nb to peer [[15336,0],4]
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:1
[cn5:14174] [[15336,1],8] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_nb: initiating connection to [[15336,0],4]
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],4]
[cn4:17831] mca: base: components_register: found loaded component binomial
[cn4:17831] mca: base: components_register: component binomial has no register or open function
[cn3:21420] mca: base: components_register: registering rml components
[cn3:21420] mca: base: components_register: found loaded component oob
[cn3:21420] mca: base: components_register: component oob has no register or open function
[cn1:14224] [[15336,1],0]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14224] [[15336,1],0] RECV CONNECT ACK FROM [[15336,0],0] ON SOCKET 11
[cn1:14224] [[15336,1],0] waiting for connect ack from [[15336,0],0]
[cn1:14224] [[15336,1],0] connect ack received from [[15336,0],0]
[cn1:14224] [[15336,1],0] connect-ack recvd from [[15336,0],0]
[cn1:14224] [[15336,1],0] connect-ack header from [[15336,0],0] is okay
[cn1:14224] [[15336,1],0] waiting for connect ack from [[15336,0],0]
[cn1:14224] [[15336,1],0] connect ack received from [[15336,0],0]
[cn1:14224] [[15336,1],0] connect-ack version from [[15336,0],0] matches ours
[cn1:14224] [[15336,1],0] connect-ack [[15336,0],0] authenticated
[cn1:14224] [[15336,1],0]-[[15336,0],0] tcp_peer_connected on socket 11
[cn1:14224] [[15336,1],0]-[[15336,0],0] connected: 192.168.1.1 - 192.168.1.1 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn1:14224] [[15336,1],0]:tcp:recv:handler starting send/recv events
[cn1:14224] [[15336,1],0] tcp:set_module called for peer [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14224] [[15336,1],0] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 75 BYTES ON SOCKET 11
[cn1:14224] [[15336,1],0]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 520
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 520 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 575
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 575 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 443
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 443 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 304
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 304 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 520 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 575 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 443 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 304 bytes from [[15336,0],2] for tag 2
[cn2:01046] mca: base: components_register: registering dstore components
[cn2:01046] mca: base: components_register: found loaded component hash
[cn2:01046] mca: base: components_register: component hash register function successful
[cn2:01046] mca: base: components_open: opening dstore components
[cn2:01046] mca: base: components_open: found loaded component hash
[cn2:01046] mca:dstore:select: checking available component hash
[cn2:01046] mca:dstore:select: Querying component [hash]
[cn5:14174] [[15336,1],8] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],4] via interface eth0
[cn5:14174] [[15336,1],8] oob:tcp:peer creating socket to [[15336,0],4]
[cn5:14174] [[15336,1],8] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],4] via interface eth0 on socket 11
[cn5:14174] [[15336,1],8] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],4] on 192.168.1.5:34732 - 0 retries
[cn5:14174] [[15336,1],8] waiting for connect completion to [[15336,0],4] - activating send event
[cn4:17831] mca: base: components_register: found loaded component debruijn
[cn4:17831] mca: base: components_register: component debruijn has no register or open function
[cn4:17831] mca: base: components_register: found loaded component direct
[cn4:17831] mca: base: components_register: component direct has no register or open function
[cn4:17831] mca: base: components_register: found loaded component radix
[cn3:21420] mca: base: components_open: opening rml components
[cn3:21420] mca: base: components_open: found loaded component oob
[cn3:21420] mca: base: components_open: component oob open function successful
[cn3:21420] orte_rml_base_select: initializing rml component oob
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 206
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 206 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 810
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 810 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1313
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1313 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 276
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 276 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 206 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 810 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 1313 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 276 bytes from [[15336,0],2] for tag 2
[cn2:01045] procdir: /tmp/openmpi-sessions-artpol@cn2_0/15336/1/2
[cn2:01045] jobdir: /tmp/openmpi-sessions-artpol@cn2_0/15336/1
[cn2:01045] top: openmpi-sessions-artpol@cn2_0
[cn5:14174] [[15336,1],8] tcp:send_handler called to send to peer [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_handler CONNECTING
[cn5:14174] [[15336,1],8]:tcp:complete_connect called for peer [[15336,0],4] on socket 11
[cn5:14174] [[15336,1],8] tcp_peer_complete_connect: sending ack to [[15336,0],4]
[cn5:14174] [[15336,1],8] SEND CONNECT ACK
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key opal.cred on proc 35364798465
[cn5:14174] dstore_hash:fetch key opal.cred for proc 35364798465 not found
[cn5:14174] [[15336,1],8] send blocking of 47 bytes to socket 11
[cn5:14174] [[15336,1],8] blocking send complete to socket 11
[cn5:14174] [[15336,1],8] tcp_peer_complete_connect: setting read event on connection to [[15336,0],4]
[cn4:17831] mca: base: components_register: component radix register function successful
[cn4:17831] mca: base: components_open: opening routed components
[cn4:17831] mca: base: components_open: found loaded component binomial
[cn4:17831] mca: base: components_open: found loaded component debruijn
[cn4:17831] mca: base: components_open: found loaded component direct
[cn4:17831] mca: base: components_open: found loaded component radix
[cn4:17831] mca:base:select: Auto-selecting routed components
[cn4:17831] mca:base:select:(routed) Querying component [binomial]
[cn4:17831] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn4:17831] mca:base:select:(routed) Querying component [debruijn]
[cn4:17831] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn4:17831] mca:base:select:(routed) Querying component [direct]
[cn4:17831] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn4:17831] mca:base:select:(routed) Querying component [radix]
[cn4:17831] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn4:17831] mca:base:select:(routed) Selected component [binomial]
[cn4:17831] mca: base: close: unloading component debruijn
[cn4:17831] mca: base: close: unloading component direct
[cn3:21419] mca: base: components_register: registering routed components
[cn3:21419] mca: base: components_register: found loaded component binomial
[cn3:21419] mca: base: components_register: component binomial has no register or open function
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 709
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 709 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1462
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1462 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 164
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 164 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 443
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 443 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 709 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 1462 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 164 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 443 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 276
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 276 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 104
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 104 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1256
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1256 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] tmp: /tmp
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: storing key opal.job.session.dir[OPAL_STRING] for proc 9594994689
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: storing key opal.my.session.dir[OPAL_STRING] for proc 9594994689
[cn2:01045] [[15336,1],2] routed_binomial: init routes for proc job [15336,1]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
	daemon uri 1005060096.1;tcp://192.168.1.2:51705
[cn2:01045] [[15336,1],2] [[15336,0],1] with daemon registering
[cn5:14174] [[15336,1],8]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14174] [[15336,1],8] RECV CONNECT ACK FROM [[15336,0],4] ON SOCKET 11
[cn5:14174] [[15336,1],8] waiting for connect ack from [[15336,0],4]
[cn5:14174] [[15336,1],8] connect ack received from [[15336,0],4]
[cn5:14174] [[15336,1],8] connect-ack recvd from [[15336,0],4]
[cn5:14174] [[15336,1],8] connect-ack header from [[15336,0],4] is okay
[cn5:14174] [[15336,1],8] waiting for connect ack from [[15336,0],4]
[cn5:14174] [[15336,1],8] connect ack received from [[15336,0],4]
[cn5:14174] [[15336,1],8] connect-ack version from [[15336,0],4] matches ours
[cn5:14174] [[15336,1],8] connect-ack [[15336,0],4] authenticated
[cn5:14174] [[15336,1],8]-[[15336,0],4] tcp_peer_connected on socket 11
[cn5:14174] [[15336,1],8]-[[15336,0],4] connected: 192.168.1.5 - 192.168.1.5 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn5:14174] [[15336,1],8]:tcp:recv:handler starting send/recv events
[cn5:14174] [[15336,1],8] tcp:set_module called for peer [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_handler called to send to peer [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_handler SENDING TO [[15336,0],4]
[cn5:14174] [[15336,1],8] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 62 BYTES ON SOCKET 11
[cn5:14174] [[15336,1],8]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17831] mca: base: close: unloading component radix
[cn4:17831] orte_routed_base_select: initializing selected component binomial
[cn3:21419] mca: base: components_register: found loaded component debruijn
[cn3:21419] mca: base: components_register: component debruijn has no register or open function
[cn3:21419] mca: base: components_register: found loaded component direct
[cn3:21419] mca: base: components_register: component direct has no register or open function
[cn3:21419] mca: base: components_register: found loaded component radix
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 960
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 960 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 276 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 104 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 1256 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 960 bytes from [[15336,0],1] for tag 2
[cn5:14175] mca: base: components_register: registering routed components
[cn5:14175] mca: base: components_register: found loaded component binomial
[cn5:14175] mca: base: components_register: component binomial has no register or open function
[cn4:17832] mca: base: components_register: registering routed components
[cn3:21419] mca: base: components_register: component radix register function successful
[cn3:21419] mca: base: components_open: opening routed components
[cn3:21419] mca: base: components_open: found loaded component binomial
[cn3:21419] mca: base: components_open: found loaded component debruijn
[cn3:21419] mca: base: components_open: found loaded component direct
[cn3:21419] mca: base: components_open: found loaded component radix
[cn3:21419] mca:base:select: Auto-selecting routed components
[cn3:21419] mca:base:select:(routed) Querying component [binomial]
[cn3:21419] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn3:21419] mca:base:select:(routed) Querying component [debruijn]
[cn3:21419] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn3:21419] mca:base:select:(routed) Querying component [direct]
[cn3:21419] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn3:21419] mca:base:select:(routed) Querying component [radix]
[cn3:21419] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn3:21419] mca:base:select:(routed) Selected component [binomial]
[cn3:21419] mca: base: close: unloading component debruijn
[cn2:01045] [[15336,1],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 20
[cn2:01045] [[15336,1],2] rml_send_buffer to peer [[15336,0],1] at tag 1
[cn2:01045] [[15336,1],2] registering sync waiting for ack
[cn2:01045] [[15336,1],2] waiting on progress thread at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/routed/base/routed_base_fns.c:350
[cn2:01045] [[15336,1],2] posting recv
[cn2:01045] [[15336,1],2] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn2:01045] [[15336,1],2] posting recv
[cn2:01045] [[15336,1],2] posting persistent recv on tag 48 for peer [[WILDCARD],WILDCARD]
[cn2:01045] [[15336,1],2] posting recv
[cn2:01045] [[15336,1],2] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn2:01045] [[15336,1],2]: set_addr to uri 1005060096.1;tcp://192.168.1.2:51705
[cn2:01045] [[15336,1],2]:set_addr checking if peer [[15336,0],1] is reachable via component tcp
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 452
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 452 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 443
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 443 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 398
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 398 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 452 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 443 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 398 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1297
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1297 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1256
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1256 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 152
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 152 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 104
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 104 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] [[15336,1],2] oob:tcp: working peer [[15336,0],1] address tcp://192.168.1.2:51705
[cn2:01045] [[15336,1],2] PEER [[15336,0],1] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn2:01045] [[15336,1],2] PASSING ADDR 192.168.1.2 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn2:01045] [[15336,1],2]:tcp set addr for peer [[15336,0],1]
[cn2:01045] [[15336,1],2]: peer [[15336,0],1] is reachable via component tcp
[cn5:14175] mca: base: components_register: found loaded component debruijn
[cn5:14175] mca: base: components_register: component debruijn has no register or open function
[cn5:14175] mca: base: components_register: found loaded component direct
[cn5:14175] mca: base: components_register: component direct has no register or open function
[cn5:14175] mca: base: components_register: found loaded component radix
[cn4:17831] mca: base: components_register: registering dstore components
[cn4:17831] mca: base: components_register: found loaded component hash
[cn4:17831] mca: base: components_register: component hash register function successful
[cn4:17831] mca: base: components_open: opening dstore components
[cn4:17831] mca: base: components_open: found loaded component hash
[cn3:21419] mca: base: close: unloading component direct
[cn3:21419] mca: base: close: unloading component radix
[cn3:21419] orte_routed_base_select: initializing selected component binomial
[cn1:14209] [[15336,0],0] message received 1297 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 1256 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 152 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 104 bytes from [[15336,0],2] for tag 2
[cn2:01045] [[15336,1],2] posting recv
[cn2:01045] [[15336,1],2] posting non-persistent recv on tag 20 for peer [[WILDCARD],WILDCARD]
[cn2:01045] [[15336,1],2] rml_send_msg to peer [[15336,0],1] at tag 1
[cn2:01045] [[15336,1],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01045] [[15336,1],2]:tcp:processing set_peer cmd for interface eth0
[cn2:01045] [[15336,1],2] SET_PEER ADDING PEER [[15336,0],1]
[cn2:01045] [[15336,1],2] set_peer: peer [[15336,0],1] is listening on interface eth0 net 192.168.1.2 port 51705
[cn2:01045] [[15336,1],2] oob:base:send to target [[15336,0],1]
[cn2:01045] [[15336,1],2] oob:tcp:send_nb to peer [[15336,0],1]:1
[cn2:01045] [[15336,1],2] tcp:send_nb to peer [[15336,0],1]
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:1
[cn2:01045] [[15336,1],2] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],1]
[cn5:14175] mca: base: components_register: component radix register function successful
[cn5:14175] mca: base: components_open: opening routed components
[cn5:14175] mca: base: components_open: found loaded component binomial
[cn5:14175] mca: base: components_open: found loaded component debruijn
[cn5:14175] mca: base: components_open: found loaded component direct
[cn5:14175] mca: base: components_open: found loaded component radix
[cn5:14175] mca:base:select: Auto-selecting routed components
[cn5:14175] mca:base:select:(routed) Querying component [binomial]
[cn5:14175] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn5:14175] mca:base:select:(routed) Querying component [debruijn]
[cn5:14175] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn5:14175] mca:base:select:(routed) Querying component [direct]
[cn5:14175] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn5:14175] mca:base:select:(routed) Querying component [radix]
[cn5:14175] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn5:14175] mca:base:select:(routed) Selected component [binomial]
[cn5:14175] mca: base: close: unloading component debruijn
[cn4:17831] mca:dstore:select: checking available component hash
[cn4:17831] mca:dstore:select: Querying component [hash]
[cn3:21419] mca: base: components_register: registering dstore components
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 689
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 689 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 106
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 106 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 102
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 102 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 689 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 106 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 102 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 118
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 118 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] [[15336,1],2] tcp:send_nb: initiating connection to [[15336,0],1]
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],1]
[cn2:01045] [[15336,1],2] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],1] via interface eth0
[cn2:01045] [[15336,1],2] oob:tcp:peer creating socket to [[15336,0],1]
[cn2:01045] [[15336,1],2] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],1] via interface eth0 on socket 11
[cn2:01045] [[15336,1],2] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],1] on 192.168.1.2:51705 - 0 retries
[cn5:14175] mca: base: close: unloading component direct
[cn5:14175] mca: base: close: unloading component radix
[cn5:14175] orte_routed_base_select: initializing selected component binomial
[cn4:17832] mca: base: components_register: found loaded component binomial
[cn3:21419] mca: base: components_register: found loaded component hash
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 477
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 477 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 118 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 477 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 126 bytes from [[15336,0],3] for tag 2
[cn3:21419] mca: base: components_register: component hash register function successful
[cn2:01045] [[15336,1],2] waiting for connect completion to [[15336,0],1] - activating send event
[cn2:01045] [[15336,1],2] tcp:send_handler called to send to peer [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_handler CONNECTING
[cn2:01045] [[15336,1],2]:tcp:complete_connect called for peer [[15336,0],1] on socket 11
[cn2:01045] [[15336,1],2] tcp_peer_complete_connect: sending ack to [[15336,0],1]
[cn2:01045] [[15336,1],2] SEND CONNECT ACK
[cn5:14175] mca: base: components_register: registering dstore components
[cn5:14175] mca: base: components_register: found loaded component hash
[cn4:17832] mca: base: components_register: component binomial has no register or open function
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 459
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 459 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 374
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 374 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 443
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 443 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 286
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 286 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 459 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 374 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 443 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 286 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 267
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 267 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1256
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1256 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 104
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 104 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key opal.cred on proc 9594994689
[cn2:01045] dstore_hash:fetch key opal.cred for proc 9594994689 not found
[cn2:01045] [[15336,1],2] send blocking of 47 bytes to socket 11
[cn2:01045] [[15336,1],2] blocking send complete to socket 11
[cn2:01045] [[15336,1],2] tcp_peer_complete_connect: setting read event on connection to [[15336,0],1]
[cn5:14175] mca: base: components_register: component hash register function successful
[cn5:14175] mca: base: components_open: opening dstore components
[cn5:14175] mca: base: components_open: found loaded component hash
[cn5:14175] mca:dstore:select: checking available component hash
[cn5:14175] mca:dstore:select: Querying component [hash]
[cn4:17832] mca: base: components_register: found loaded component debruijn
[cn4:17832] mca: base: components_register: component debruijn has no register or open function
[cn4:17832] mca: base: components_register: found loaded component direct
[cn4:17832] mca: base: components_register: component direct has no register or open function
[cn4:17832] mca: base: components_register: found loaded component radix
[cn3:21419] mca: base: components_open: opening dstore components
[cn3:21419] mca: base: components_open: found loaded component hash
[cn3:21419] mca:dstore:select: checking available component hash
[cn3:21419] mca:dstore:select: Querying component [hash]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1217
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1217 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 267 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 1256 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 104 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 1217 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 738
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 738 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 106
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 106 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14175] mca: base: components_register: registering grpcomm components
[cn5:14175] mca: base: components_register: found loaded component bad
[cn5:14175] mca: base: components_register: component bad has no register or open function
[cn4:17832] mca: base: components_register: component radix register function successful
[cn4:17832] mca: base: components_open: opening routed components
[cn4:17832] mca: base: components_open: found loaded component binomial
[cn4:17832] mca: base: components_open: found loaded component debruijn
[cn4:17832] mca: base: components_open: found loaded component direct
[cn4:17832] mca: base: components_open: found loaded component radix
[cn4:17832] mca:base:select: Auto-selecting routed components
[cn4:17832] mca:base:select:(routed) Querying component [binomial]
[cn4:17832] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn4:17832] mca:base:select:(routed) Querying component [debruijn]
[cn4:17832] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn4:17832] mca:base:select:(routed) Querying component [direct]
[cn4:17832] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn4:17832] mca:base:select:(routed) Querying component [radix]
[cn4:17832] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn4:17832] mca:base:select:(routed) Selected component [binomial]
[cn4:17832] mca: base: close: unloading component debruijn
[cn3:21420] mca: base: components_register: registering routed components
[cn2:01045] [[15336,1],2]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01045] [[15336,1],2] RECV CONNECT ACK FROM [[15336,0],1] ON SOCKET 11
[cn2:01045] [[15336,1],2] waiting for connect ack from [[15336,0],1]
[cn2:01045] [[15336,1],2] connect ack received from [[15336,0],1]
[cn2:01045] [[15336,1],2] connect-ack recvd from [[15336,0],1]
[cn2:01045] [[15336,1],2] connect-ack header from [[15336,0],1] is okay
[cn2:01045] [[15336,1],2] waiting for connect ack from [[15336,0],1]
[cn2:01045] [[15336,1],2] connect ack received from [[15336,0],1]
[cn2:01045] [[15336,1],2] connect-ack version from [[15336,0],1] matches ours
[cn2:01045] [[15336,1],2] connect-ack [[15336,0],1] authenticated
[cn2:01045] [[15336,1],2]-[[15336,0],1] tcp_peer_connected on socket 11
[cn2:01045] [[15336,1],2]-[[15336,0],1] connected: 192.168.1.2 - 192.168.1.2 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn2:01045] [[15336,1],2]:tcp:recv:handler starting send/recv events
[cn2:01045] [[15336,1],2] tcp:set_module called for peer [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_handler called to send to peer [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_handler SENDING TO [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 275
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 275 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 738 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 106 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 275 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 267
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 267 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 190
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 190 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 267
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 267 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14175] mca: base: components_open: opening grpcomm components
[cn5:14175] mca: base: components_open: found loaded component bad
[cn5:14175] mca: base: components_open: component bad open function successful
[cn5:14175] mca:base:select: Auto-selecting grpcomm components
[cn5:14175] mca:base:select:(grpcomm) Querying component [bad]
[cn5:14175] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn5:14175] mca:base:select:(grpcomm) Selected component [bad]
[cn5:14175] [[15336,1],9] grpcomm:base:receive start comm
[cn5:14175] [[15336,1],9] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn5:14175] [[15336,1],9] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 48
[cn4:17832] mca: base: close: unloading component direct
[cn4:17832] mca: base: close: unloading component radix
[cn4:17832] orte_routed_base_select: initializing selected component binomial
[cn3:21420] mca: base: components_register: found loaded component binomial
[cn2:01045] [[15336,1],2] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 62 BYTES ON SOCKET 11
[cn2:01045] [[15336,1],2]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 267 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 190 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 267 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 126 bytes from [[15336,0],2] for tag 2
[cn2:01046] mca: base: components_register: registering grpcomm components
[cn2:01046] mca: base: components_register: found loaded component bad
[cn2:01046] mca: base: components_register: component bad has no register or open function
[cn5:14175] [[15336,1],9] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn5:14175] [[15336,1],9] setting up session dir with
	tmpdir: /tmp
	host cn5
[cn4:17831] mca: base: components_register: registering grpcomm components
[cn4:17831] mca: base: components_register: found loaded component bad
[cn4:17831] mca: base: components_register: component bad has no register or open function
[cn3:21420] mca: base: components_register: component binomial has no register or open function
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 738
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 738 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 160
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 160 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 104
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 104 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 443
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 443 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 738 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 160 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 104 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 443 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 190
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 190 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 289
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 289 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 898
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 898 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1256
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1256 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] mca: base: components_open: opening grpcomm components
[cn2:01046] mca: base: components_open: found loaded component bad
[cn2:01046] mca: base: components_open: component bad open function successful
[cn2:01046] mca:base:select: Auto-selecting grpcomm components
[cn2:01046] mca:base:select:(grpcomm) Querying component [bad]
[cn2:01046] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn2:01046] mca:base:select:(grpcomm) Selected component [bad]
[cn2:01046] [[15336,1],3] grpcomm:base:receive start comm
[cn2:01046] [[15336,1],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn2:01046] [[15336,1],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 48
[cn5:14175] [[15336,1],9] posting recv
[cn5:14175] [[15336,1],9] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn4:17832] mca: base: components_register: registering dstore components
[cn3:21420] mca: base: components_register: found loaded component debruijn
[cn3:21420] mca: base: components_register: component debruijn has no register or open function
[cn3:21420] mca: base: components_register: found loaded component direct
[cn3:21420] mca: base: components_register: component direct has no register or open function
[cn3:21420] mca: base: components_register: found loaded component radix
[cn1:14209] [[15336,0],0] message received 190 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 289 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 898 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 1256 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 336
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 336 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 102
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 102 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn2:01046] [[15336,1],3] setting up session dir with
	tmpdir: /tmp
	host cn2
[cn5:14175] [[15336,1],9] posting recv
[cn5:14175] [[15336,1],9] posting persistent recv on tag 48 for peer [[WILDCARD],WILDCARD]
[cn5:14175] [[15336,1],9] posting recv
[cn5:14175] [[15336,1],9] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn4:17831] mca: base: components_open: opening grpcomm components
[cn4:17831] mca: base: components_open: found loaded component bad
[cn4:17831] mca: base: components_open: component bad open function successful
[cn4:17831] mca:base:select: Auto-selecting grpcomm components
[cn4:17831] mca:base:select:(grpcomm) Querying component [bad]
[cn4:17831] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn4:17831] mca:base:select:(grpcomm) Selected component [bad]
[cn4:17831] [[15336,1],6] grpcomm:base:receive start comm
[cn4:17831] [[15336,1],6] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn4:17831] [[15336,1],6] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 48
[cn4:17831] [[15336,1],6] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn4:17831] [[15336,1],6] setting up session dir with
	tmpdir: /tmp
	host cn4
[cn3:21420] mca: base: components_register: component radix register function successful
[cn3:21420] mca: base: components_open: opening routed components
[cn3:21420] mca: base: components_open: found loaded component binomial
[cn3:21420] mca: base: components_open: found loaded component debruijn
[cn3:21420] mca: base: components_open: found loaded component direct
[cn3:21420] mca: base: components_open: found loaded component radix
[cn3:21420] mca:base:select: Auto-selecting routed components
[cn3:21420] mca:base:select:(routed) Querying component [binomial]
[cn3:21420] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn3:21420] mca:base:select:(routed) Querying component [debruijn]
[cn3:21420] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn3:21420] mca:base:select:(routed) Querying component [direct]
[cn3:21420] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn3:21420] mca:base:select:(routed) Querying component [radix]
[cn3:21420] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn3:21420] mca:base:select:(routed) Selected component [binomial]
[cn3:21420] mca: base: close: unloading component debruijn
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 160
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 160 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 336 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 102 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 160 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 289
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 289 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 739
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 739 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 118
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 118 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 105
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 105 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14175] procdir: /tmp/openmpi-sessions-artpol@cn5_0/15336/1/9
[cn5:14175] jobdir: /tmp/openmpi-sessions-artpol@cn5_0/15336/1
[cn5:14175] top: openmpi-sessions-artpol@cn5_0
[cn5:14175] tmp: /tmp
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn4:17832] mca: base: components_register: found loaded component hash
[cn3:21420] mca: base: close: unloading component direct
[cn3:21420] mca: base: close: unloading component radix
[cn3:21420] orte_routed_base_select: initializing selected component binomial
[cn2:01046] [[15336,1],3] posting recv
[cn2:01046] [[15336,1],3] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] message received 289 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 739 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 118 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 105 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 228
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 228 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 876
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 876 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 160
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 160 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 192
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 192 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] posting recv
[cn2:01046] [[15336,1],3] posting persistent recv on tag 48 for peer [[WILDCARD],WILDCARD]
[cn2:01046] [[15336,1],3] posting recv
[cn2:01046] [[15336,1],3] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn5:14175] dstore:hash:store: storing key opal.job.session.dir[OPAL_STRING] for proc 39659765761
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: storing key opal.my.session.dir[OPAL_STRING] for proc 39659765761
[cn5:14175] [[15336,1],9] routed_binomial: init routes for proc job [15336,1]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
	daemon uri 1005060096.4;tcp://192.168.1.5:34732
[cn5:14175] [[15336,1],9] [[15336,0],4] with daemon registering
[cn5:14175] [[15336,1],9] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 20
[cn5:14175] [[15336,1],9] rml_send_buffer to peer [[15336,0],4] at tag 1
[cn4:17832] mca: base: components_register: component hash register function successful
[cn3:21419] mca: base: components_register: registering grpcomm components
[cn1:14209] [[15336,0],0] message received 228 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 876 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 160 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 192 bytes from [[15336,0],2] for tag 2
[cn2:01046] procdir: /tmp/openmpi-sessions-artpol@cn2_0/15336/1/3
[cn2:01046] jobdir: /tmp/openmpi-sessions-artpol@cn2_0/15336/1
[cn2:01046] top: openmpi-sessions-artpol@cn2_0
[cn2:01046] tmp: /tmp
[cn5:14175] [[15336,1],9] registering sync waiting for ack
[cn5:14175] [[15336,1],9] waiting on progress thread at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/routed/base/routed_base_fns.c:350
[cn5:14175] [[15336,1],9]: set_addr to uri 1005060096.4;tcp://192.168.1.5:34732
[cn5:14175] [[15336,1],9]:set_addr checking if peer [[15336,0],4] is reachable via component tcp
[cn5:14175] [[15336,1],9] oob:tcp: working peer [[15336,0],4] address tcp://192.168.1.5:34732
[cn5:14175] [[15336,1],9] PEER [[15336,0],4] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn5:14175] [[15336,1],9] PASSING ADDR 192.168.1.5 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn5:14175] [[15336,1],9]:tcp set addr for peer [[15336,0],4]
[cn5:14175] [[15336,1],9]: peer [[15336,0],4] is reachable via component tcp
[cn5:14175] [[15336,1],9] posting recv
[cn4:17831] [[15336,1],6] posting recv
[cn4:17831] [[15336,1],6] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn3:21419] mca: base: components_register: found loaded component bad
[cn3:21419] mca: base: components_register: component bad has no register or open function
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 691
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 691 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1586
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1586 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 289
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 289 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 738
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 738 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 691 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 1586 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 289 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 738 bytes from [[15336,0],2] for tag 2
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: storing key opal.job.session.dir[OPAL_STRING] for proc 13889961985
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: storing key opal.my.session.dir[OPAL_STRING] for proc 13889961985
[cn2:01046] [[15336,1],3] routed_binomial: init routes for proc job [15336,1]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
	daemon uri 1005060096.1;tcp://192.168.1.2:51705
[cn2:01046] [[15336,1],3] [[15336,0],1] with daemon registering
[cn5:14175] [[15336,1],9] posting non-persistent recv on tag 20 for peer [[WILDCARD],WILDCARD]
[cn5:14175] [[15336,1],9] rml_send_msg to peer [[15336,0],4] at tag 1
[cn5:14175] [[15336,1],9] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14175] [[15336,1],9]:tcp:processing set_peer cmd for interface eth0
[cn5:14175] [[15336,1],9] SET_PEER ADDING PEER [[15336,0],4]
[cn5:14175] [[15336,1],9] set_peer: peer [[15336,0],4] is listening on interface eth0 net 192.168.1.5 port 34732
[cn5:14175] [[15336,1],9] oob:base:send to target [[15336,0],4]
[cn5:14175] [[15336,1],9] oob:tcp:send_nb to peer [[15336,0],4]:1
[cn5:14175] [[15336,1],9] tcp:send_nb to peer [[15336,0],4]
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:1
[cn5:14175] [[15336,1],9] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_nb: initiating connection to [[15336,0],4]
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],4]
[cn5:14175] [[15336,1],9] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],4] via interface eth0
[cn4:17831] [[15336,1],6] posting recv
[cn4:17831] [[15336,1],6] posting persistent recv on tag 48 for peer [[WILDCARD],WILDCARD]
[cn4:17831] [[15336,1],6] posting recv
[cn4:17831] [[15336,1],6] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn3:21419] mca: base: components_open: opening grpcomm components
[cn3:21419] mca: base: components_open: found loaded component bad
[cn3:21419] mca: base: components_open: component bad open function successful
[cn3:21419] mca:base:select: Auto-selecting grpcomm components
[cn3:21419] mca:base:select:(grpcomm) Querying component [bad]
[cn3:21419] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn3:21419] mca:base:select:(grpcomm) Selected component [bad]
[cn3:21419] [[15336,1],4] grpcomm:base:receive start comm
[cn3:21419] [[15336,1],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn3:21419] [[15336,1],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 48
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1197
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1197 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 459
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 459 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 286
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 286 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 190
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 190 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1197 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 459 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 286 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 190 bytes from [[15336,0],2] for tag 2
[cn2:01046] [[15336,1],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 20
[cn2:01046] [[15336,1],3] rml_send_buffer to peer [[15336,0],1] at tag 1
[cn2:01046] [[15336,1],3] registering sync waiting for ack
[cn2:01046] [[15336,1],3] waiting on progress thread at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/routed/base/routed_base_fns.c:350
[cn2:01046] [[15336,1],3]: set_addr to uri 1005060096.1;tcp://192.168.1.2:51705
[cn2:01046] [[15336,1],3]:set_addr checking if peer [[15336,0],1] is reachable via component tcp
[cn2:01046] [[15336,1],3] oob:tcp: working peer [[15336,0],1] address tcp://192.168.1.2:51705
[cn2:01046] [[15336,1],3] PEER [[15336,0],1] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn2:01046] [[15336,1],3] PASSING ADDR 192.168.1.2 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn2:01046] [[15336,1],3]:tcp set addr for peer [[15336,0],1]
[cn2:01046] [[15336,1],3]: peer [[15336,0],1] is reachable via component tcp
[cn2:01046] [[15336,1],3] posting recv
[cn2:01046] [[15336,1],3] posting non-persistent recv on tag 20 for peer [[WILDCARD],WILDCARD]
[cn2:01046] [[15336,1],3] rml_send_msg to peer [[15336,0],1] at tag 1
[cn5:14175] [[15336,1],9] oob:tcp:peer creating socket to [[15336,0],4]
[cn5:14175] [[15336,1],9] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],4] via interface eth0 on socket 11
[cn5:14175] [[15336,1],9] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],4] on 192.168.1.5:34732 - 0 retries
[cn5:14175] [[15336,1],9] waiting for connect completion to [[15336,0],4] - activating send event
[cn4:17832] mca: base: components_open: opening dstore components
[cn4:17832] mca: base: components_open: found loaded component hash
[cn4:17832] mca:dstore:select: checking available component hash
[cn4:17832] mca:dstore:select: Querying component [hash]
[cn3:21419] [[15336,1],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn3:21419] [[15336,1],4] setting up session dir with
	tmpdir: /tmp
	host cn3
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 725
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 725 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 810
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 810 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1253
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1253 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 104
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 104 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 725 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 810 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 1253 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 104 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1289
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1289 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1462
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1462 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 110
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 110 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 102
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 102 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01046] [[15336,1],3]:tcp:processing set_peer cmd for interface eth0
[cn2:01046] [[15336,1],3] SET_PEER ADDING PEER [[15336,0],1]
[cn2:01046] [[15336,1],3] set_peer: peer [[15336,0],1] is listening on interface eth0 net 192.168.1.2 port 51705
[cn2:01046] [[15336,1],3] oob:base:send to target [[15336,0],1]
[cn2:01046] [[15336,1],3] oob:tcp:send_nb to peer [[15336,0],1]:1
[cn2:01046] [[15336,1],3] tcp:send_nb to peer [[15336,0],1]
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn5:14175] [[15336,1],9] tcp:send_handler called to send to peer [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_handler CONNECTING
[cn5:14175] [[15336,1],9]:tcp:complete_connect called for peer [[15336,0],4] on socket 11
[cn5:14175] [[15336,1],9] tcp_peer_complete_connect: sending ack to [[15336,0],4]
[cn5:14175] [[15336,1],9] SEND CONNECT ACK
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key opal.cred on proc 39659765761
[cn5:14175] dstore_hash:fetch key opal.cred for proc 39659765761 not found
[cn5:14175] [[15336,1],9] send blocking of 47 bytes to socket 11
[cn5:14175] [[15336,1],9] blocking send complete to socket 11
[cn5:14175] [[15336,1],9] tcp_peer_complete_connect: setting read event on connection to [[15336,0],4]
[cn4:17831] procdir: /tmp/openmpi-sessions-artpol@cn4_0/15336/1/6
[cn4:17831] jobdir: /tmp/openmpi-sessions-artpol@cn4_0/15336/1
[cn4:17831] top: openmpi-sessions-artpol@cn4_0
[cn4:17831] tmp: /tmp
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key opal.job.session.dir[OPAL_STRING] for proc 26774863873
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key opal.my.session.dir[OPAL_STRING] for proc 26774863873
[cn4:17831] [[15336,1],6] routed_binomial: init routes for proc job [15336,1]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
	daemon uri 1005060096.3;tcp://192.168.1.4:36268
[cn4:17831] [[15336,1],6] [[15336,0],3] with daemon registering
[cn4:17831] [[15336,1],6] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 20
[cn4:17831] [[15336,1],6] rml_send_buffer to peer [[15336,0],3] at tag 1
[cn4:17831] [[15336,1],6] registering sync waiting for ack
[cn4:17831] [[15336,1],6] waiting on progress thread at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/routed/base/routed_base_fns.c:350
[cn3:21420] mca: base: components_register: registering dstore components
[cn1:14209] [[15336,0],0] message received 1289 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 1462 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 110 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 102 bytes from [[15336,0],2] for tag 2
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:1
[cn2:01046] [[15336,1],3] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_nb: initiating connection to [[15336,0],1]
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],1]
[cn2:01046] [[15336,1],3] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],1] via interface eth0
[cn2:01046] [[15336,1],3] oob:tcp:peer creating socket to [[15336,0],1]
[cn2:01046] [[15336,1],3] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],1] via interface eth0 on socket 11
[cn2:01046] [[15336,1],3] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],1] on 192.168.1.2:51705 - 0 retries
[cn2:01046] [[15336,1],3] waiting for connect completion to [[15336,0],1] - activating send event
[cn2:01046] [[15336,1],3] tcp:send_handler called to send to peer [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_handler CONNECTING
[cn5:14175] [[15336,1],9]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14175] [[15336,1],9] RECV CONNECT ACK FROM [[15336,0],4] ON SOCKET 11
[cn5:14175] [[15336,1],9] waiting for connect ack from [[15336,0],4]
[cn5:14175] [[15336,1],9] connect ack received from [[15336,0],4]
[cn5:14175] [[15336,1],9] connect-ack recvd from [[15336,0],4]
[cn5:14175] [[15336,1],9] connect-ack header from [[15336,0],4] is okay
[cn5:14175] [[15336,1],9] waiting for connect ack from [[15336,0],4]
[cn5:14175] [[15336,1],9] connect ack received from [[15336,0],4]
[cn5:14175] [[15336,1],9] connect-ack version from [[15336,0],4] matches ours
[cn5:14175] [[15336,1],9] connect-ack [[15336,0],4] authenticated
[cn5:14175] [[15336,1],9]-[[15336,0],4] tcp_peer_connected on socket 11
[cn5:14175] [[15336,1],9]-[[15336,0],4] connected: 192.168.1.5 - 192.168.1.5 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn5:14175] [[15336,1],9]:tcp:recv:handler starting send/recv events
[cn5:14175] [[15336,1],9] tcp:set_module called for peer [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_handler called to send to peer [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_handler SENDING TO [[15336,0],4]
[cn5:14175] [[15336,1],9] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 62 BYTES ON SOCKET 11
[cn5:14175] [[15336,1],9]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17831] [[15336,1],6]: set_addr to uri 1005060096.3;tcp://192.168.1.4:36268
[cn3:21420] mca: base: components_register: found loaded component hash
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 676
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 676 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 273
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 273 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 753
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 753 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 419
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 419 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 676 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 273 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 753 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 419 bytes from [[15336,0],2] for tag 2
[cn2:01046] [[15336,1],3]:tcp:complete_connect called for peer [[15336,0],1] on socket 11
[cn2:01046] [[15336,1],3] tcp_peer_complete_connect: sending ack to [[15336,0],1]
[cn2:01046] [[15336,1],3] SEND CONNECT ACK
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key opal.cred on proc 13889961985
[cn2:01046] dstore_hash:fetch key opal.cred for proc 13889961985 not found
[cn2:01046] [[15336,1],3] send blocking of 47 bytes to socket 11
[cn2:01046] [[15336,1],3] blocking send complete to socket 11
[cn2:01046] [[15336,1],3] tcp_peer_complete_connect: setting read event on connection to [[15336,0],1]
[cn5:14174] [[15336,1],8]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14174] [[15336,1],8]:tcp:recv:handler CONNECTED
[cn5:14174] [[15336,1],8]:tcp:recv:handler allocate new recv msg
[cn5:14174] [[15336,1],8]:tcp:recv:handler read hdr
[cn4:17831] [[15336,1],6]:set_addr checking if peer [[15336,0],3] is reachable via component tcp
[cn4:17831] [[15336,1],6] oob:tcp: working peer [[15336,0],3] address tcp://192.168.1.4:36268
[cn4:17831] [[15336,1],6] PEER [[15336,0],3] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn4:17831] [[15336,1],6] PASSING ADDR 192.168.1.4 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn4:17831] [[15336,1],6]:tcp set addr for peer [[15336,0],3]
[cn4:17831] [[15336,1],6]: peer [[15336,0],3] is reachable via component tcp
[cn4:17831] [[15336,1],6] posting recv
[cn4:17831] [[15336,1],6] posting non-persistent recv on tag 20 for peer [[WILDCARD],WILDCARD]
[cn4:17831] [[15336,1],6] rml_send_msg to peer [[15336,0],3] at tag 1
[cn3:21419] [[15336,1],4] posting recv
[cn3:21419] [[15336,1],4] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn3:21419] [[15336,1],4] posting recv
[cn3:21419] [[15336,1],4] posting persistent recv on tag 48 for peer [[WILDCARD],WILDCARD]
[cn3:21419] [[15336,1],4] posting recv
[cn3:21419] [[15336,1],4] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 103
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 103 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 108
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 108 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 953
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 953 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 118
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 118 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 103 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 108 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 953 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 118 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1144
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1144 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 829
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 829 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1253
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1253 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14174] [[15336,1],8]:tcp:recv:handler allocate data region of size 14140
[cn4:17831] [[15336,1],6] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17831] [[15336,1],6]:tcp:processing set_peer cmd for interface eth0
[cn4:17831] [[15336,1],6] SET_PEER ADDING PEER [[15336,0],3]
[cn4:17831] [[15336,1],6] set_peer: peer [[15336,0],3] is listening on interface eth0 net 192.168.1.4 port 36268
[cn4:17831] [[15336,1],6] oob:base:send to target [[15336,0],3]
[cn4:17831] [[15336,1],6] oob:tcp:send_nb to peer [[15336,0],3]:1
[cn4:17831] [[15336,1],6] tcp:send_nb to peer [[15336,0],3]
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:1
[cn4:17831] [[15336,1],6] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn3:21420] mca: base: components_register: component hash register function successful
[cn1:14209] [[15336,0],0] message received 1144 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] routed_binomial: init routes for HNP job [15336,1]
[cn1:14209] [[15336,0],0] routed_binomial:callback got uri 1005060097.8;tcp://192.168.1.5:59276 for job [15336,1] rank 8
[cn1:14209] [[15336,0],0] routed_binomial:callback got uri 1005060097.9;tcp://192.168.1.5:47378 for job [15336,1] rank 9
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn1:14209] [[15336,0],0] message received 829 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 1253 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 275
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 275 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 351
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 351 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 743
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 743 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 286
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 286 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] RECV CONNECT ACK FROM [[15336,0],1] ON SOCKET 11
[cn2:01046] [[15336,1],3] waiting for connect ack from [[15336,0],1]
[cn2:01046] [[15336,1],3] connect ack received from [[15336,0],1]
[cn2:01046] [[15336,1],3] connect-ack recvd from [[15336,0],1]
[cn2:01046] [[15336,1],3] connect-ack header from [[15336,0],1] is okay
[cn2:01046] [[15336,1],3] waiting for connect ack from [[15336,0],1]
[cn2:01046] [[15336,1],3] connect ack received from [[15336,0],1]
[cn2:01046] [[15336,1],3] connect-ack version from [[15336,0],1] matches ours
[cn2:01046] [[15336,1],3] connect-ack [[15336,0],1] authenticated
[cn2:01046] [[15336,1],3]-[[15336,0],1] tcp_peer_connected on socket 11
[cn2:01046] [[15336,1],3]-[[15336,0],1] connected: 192.168.1.2 - 192.168.1.2 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn2:01046] [[15336,1],3]:tcp:recv:handler starting send/recv events
[cn2:01046] [[15336,1],3] tcp:set_module called for peer [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_handler called to send to peer [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_handler SENDING TO [[15336,0],1]
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],3]
[cn4:17831] [[15336,1],6] tcp:send_nb: initiating connection to [[15336,0],3]
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],3]
[cn4:17831] [[15336,1],6] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],3] via interface eth0
[cn4:17831] [[15336,1],6] oob:tcp:peer creating socket to [[15336,0],3]
[cn4:17831] [[15336,1],6] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],3] via interface eth0 on socket 11
[cn4:17831] [[15336,1],6] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],3] on 192.168.1.4:36268 - 0 retries
[cn3:21419] procdir: /tmp/openmpi-sessions-artpol@cn3_0/15336/1/4
[cn3:21419] jobdir: /tmp/openmpi-sessions-artpol@cn3_0/15336/1
[cn3:21419] top: openmpi-sessions-artpol@cn3_0
[cn3:21419] tmp: /tmp
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key opal.job.session.dir[OPAL_STRING] for proc 18184929281
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key opal.my.session.dir[OPAL_STRING] for proc 18184929281
[cn3:21419] [[15336,1],4] routed_binomial: init routes for proc job [15336,1]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
	daemon uri 1005060096.2;tcp://192.168.1.3:42008
[cn3:21419] [[15336,1],4] [[15336,0],2] with daemon registering
[cn3:21419] [[15336,1],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 20
[cn3:21419] [[15336,1],4] rml_send_buffer to peer [[15336,0],2] at tag 1
[cn3:21419] [[15336,1],4] registering sync waiting for ack
[cn3:21419] [[15336,1],4] waiting on progress thread at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/routed/base/routed_base_fns.c:350
[cn1:14209] [[15336,0],0] message received 275 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 351 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 743 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 286 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 608
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 608 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 195
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 195 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 110
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 110 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 62 BYTES ON SOCKET 11
[cn2:01046] [[15336,1],3]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14175] [[15336,1],9]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14175] [[15336,1],9]:tcp:recv:handler CONNECTED
[cn5:14175] [[15336,1],9]:tcp:recv:handler allocate new recv msg
[cn5:14175] [[15336,1],9]:tcp:recv:handler read hdr
[cn5:14175] [[15336,1],9]:tcp:recv:handler allocate data region of size 14140
[cn4:17831] [[15336,1],6] waiting for connect completion to [[15336,0],3] - activating send event
[cn4:17831] [[15336,1],6] tcp:send_handler called to send to peer [[15336,0],3]
[cn4:17831] [[15336,1],6] tcp:send_handler CONNECTING
[cn4:17831] [[15336,1],6]:tcp:complete_connect called for peer [[15336,0],3] on socket 11
[cn4:17831] [[15336,1],6] tcp_peer_complete_connect: sending ack to [[15336,0],3]
[cn4:17831] [[15336,1],6] SEND CONNECT ACK
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key opal.cred on proc 26774863873
[cn4:17831] dstore_hash:fetch key opal.cred for proc 26774863873 not found
[cn4:17831] [[15336,1],6] send blocking of 47 bytes to socket 11
[cn3:21420] mca: base: components_open: opening dstore components
[cn3:21420] mca: base: components_open: found loaded component hash
[cn3:21420] mca:dstore:select: checking available component hash
[cn3:21420] mca:dstore:select: Querying component [hash]
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 608 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 195 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 110 bytes from [[15336,0],2] for tag 2
[cn2:01045] [[15336,1],2]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01045] [[15336,1],2]:tcp:recv:handler CONNECTED
[cn2:01045] [[15336,1],2]:tcp:recv:handler allocate new recv msg
[cn5:14174] [[15336,1],8]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14174] [[15336,1],8]:tcp:recv:handler CONNECTED
[cn5:14174] [[15336,1],8] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 14140 BYTES FOR DEST [[15336,1],8] TAG 20
[cn5:14174] [[15336,1],8] DELIVERING TO RML
[cn5:14174] [[15336,1],8] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14174] [[15336,1],8] message received 14140 bytes from [[15336,0],4] for tag 20
[cn5:14174] [[15336,1],8] registering sync ack recvd
[cn4:17831] [[15336,1],6] blocking send complete to socket 11
[cn4:17831] [[15336,1],6] tcp_peer_complete_connect: setting read event on connection to [[15336,0],3]
[cn3:21419] [[15336,1],4]: set_addr to uri 1005060096.2;tcp://192.168.1.3:42008
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 160
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 160 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 838
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 838 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1069
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1069 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 875
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 875 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 160 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 838 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 1069 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 875 bytes from [[15336,0],2] for tag 2
[cn2:01045] [[15336,1],2]:tcp:recv:handler read hdr
[cn2:01045] [[15336,1],2]:tcp:recv:handler allocate data region of size 14140
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 1005060096
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060096
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18446744069414584318
[cn5:14174] dstore:hash:store: storing key cn1[OPAL_UINT32] for proc 18446744069414584318
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 5300027392
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027392
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18446744069414584318
[cn5:14174] dstore:hash:store: storing key cn2[OPAL_UINT32] for proc 18446744069414584318
[cn4:17831] [[15336,1],6]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17831] [[15336,1],6] RECV CONNECT ACK FROM [[15336,0],3] ON SOCKET 11
[cn4:17831] [[15336,1],6] waiting for connect ack from [[15336,0],3]
[cn4:17831] [[15336,1],6] connect ack received from [[15336,0],3]
[cn4:17831] [[15336,1],6] connect-ack recvd from [[15336,0],3]
[cn4:17831] [[15336,1],6] connect-ack header from [[15336,0],3] is okay
[cn4:17831] [[15336,1],6] waiting for connect ack from [[15336,0],3]
[cn4:17831] [[15336,1],6] connect ack received from [[15336,0],3]
[cn4:17831] [[15336,1],6] connect-ack version from [[15336,0],3] matches ours
[cn4:17831] [[15336,1],6] connect-ack [[15336,0],3] authenticated
[cn4:17831] [[15336,1],6]-[[15336,0],3] tcp_peer_connected on socket 11
[cn4:17831] [[15336,1],6]-[[15336,0],3] connected: 192.168.1.4 - 192.168.1.4 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn4:17831] [[15336,1],6]:tcp:recv:handler starting send/recv events
[cn4:17831] [[15336,1],6] tcp:set_module called for peer [[15336,0],3]
[cn3:21419] [[15336,1],4]:set_addr checking if peer [[15336,0],2] is reachable via component tcp
[cn3:21419] [[15336,1],4] oob:tcp: working peer [[15336,0],2] address tcp://192.168.1.3:42008
[cn3:21419] [[15336,1],4] PEER [[15336,0],2] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn3:21419] [[15336,1],4] PASSING ADDR 192.168.1.3 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn3:21419] [[15336,1],4]:tcp set addr for peer [[15336,0],2]
[cn3:21419] [[15336,1],4]: peer [[15336,0],2] is reachable via component tcp
[cn3:21419] [[15336,1],4] posting recv
[cn3:21419] [[15336,1],4] posting non-persistent recv on tag 20 for peer [[WILDCARD],WILDCARD]
[cn3:21419] [[15336,1],4] rml_send_msg to peer [[15336,0],2] at tag 1
[cn3:21419] [[15336,1],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2905
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 2905 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 423
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 423 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1049
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1049 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],1] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] routed_binomial: init routes for HNP job [15336,1]
[cn1:14209] [[15336,0],0] routed_binomial:callback got uri 1005060097.2;tcp://192.168.1.2:39416 for job [15336,1] rank 2
[cn1:14209] [[15336,0],0] routed_binomial:callback got uri 1005060097.3;tcp://192.168.1.2:37264 for job [15336,1] rank 3
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn1:14209] [[15336,0],0] message received 2905 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 423 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 1049 bytes from [[15336,0],2] for tag 2
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 9594994688
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994688
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18446744069414584318
[cn5:14174] dstore:hash:store: storing key cn3[OPAL_UINT32] for proc 18446744069414584318
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 13889961984
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961984
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18446744069414584318
[cn5:14174] dstore:hash:store: storing key cn4[OPAL_UINT32] for proc 18446744069414584318
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18184929280
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929280
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18446744069414584318
[cn5:14174] dstore:hash:store: storing key cn5[OPAL_UINT32] for proc 18446744069414584318
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 35364798465
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 35364798465
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18446744070419644417
[cn5:14174] dstore:hash:store: storing key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18446744070419644417
[cn5:14174] dstore:hash:store: updating key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 1005060097
[cn5:14174] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 1005060097
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 1005060097
[cn5:14174] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 1005060097
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 1005060097
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 1005060097
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn5:14174] storing data in NONPEER dstore
[cn4:17831] [[15336,1],6] tcp:send_handler called to send to peer [[15336,0],3]
[cn4:17831] [[15336,1],6] tcp:send_handler SENDING TO [[15336,0],3]
[cn4:17831] [[15336,1],6] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 62 BYTES ON SOCKET 11
[cn4:17831] [[15336,1],6]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21419] [[15336,1],4]:tcp:processing set_peer cmd for interface eth0
[cn3:21419] [[15336,1],4] SET_PEER ADDING PEER [[15336,0],2]
[cn3:21419] [[15336,1],4] set_peer: peer [[15336,0],2] is listening on interface eth0 net 192.168.1.3 port 42008
[cn3:21419] [[15336,1],4] oob:base:send to target [[15336,0],2]
[cn3:21419] [[15336,1],4] oob:tcp:send_nb to peer [[15336,0],2]:1
[cn3:21419] [[15336,1],4] tcp:send_nb to peer [[15336,0],2]
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:1
[cn3:21419] [[15336,1],4] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_nb: initiating connection to [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 273
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 273 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2221
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 2221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 267
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 267 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 611
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 611 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 273 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 2221 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 267 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 611 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 108
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 108 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2771
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 2771 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 738
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 738 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 743
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 743 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01046] [[15336,1],3]:tcp:recv:handler CONNECTED
[cn2:01046] [[15336,1],3]:tcp:recv:handler allocate new recv msg
[cn2:01046] [[15336,1],3]:tcp:recv:handler read hdr
[cn5:14174] dstore:hash:store storing data for proc 1005060097
[cn5:14174] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 1005060097
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 1005060097
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060097
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 1005060097
[cn5:14174] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 1005060097
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 5300027393
[cn5:14174] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 5300027393
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 5300027393
[cn5:14174] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 5300027393
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 5300027393
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 5300027393
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 5300027393
[cn5:14174] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 5300027393
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 5300027393
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027393
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 5300027393
[cn5:14174] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 5300027393
[cn4:17832] mca: base: components_register: registering grpcomm components
[cn4:17832] mca: base: components_register: found loaded component bad
[cn4:17832] mca: base: components_register: component bad has no register or open function
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],2]
[cn3:21419] [[15336,1],4] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],2] via interface eth0
[cn3:21419] [[15336,1],4] oob:tcp:peer creating socket to [[15336,0],2]
[cn3:21419] [[15336,1],4] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],2] via interface eth0 on socket 11
[cn3:21419] [[15336,1],4] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],2] on 192.168.1.3:42008 - 0 retries
[cn1:14209] [[15336,0],0] message received 108 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 2771 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 738 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 743 bytes from [[15336,0],2] for tag 2
[cn2:01046] [[15336,1],3]:tcp:recv:handler allocate data region of size 14140
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 9594994689
[cn5:14174] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 9594994689
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 9594994689
[cn5:14174] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 9594994689
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 9594994689
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 9594994689
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 9594994689
[cn5:14174] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 9594994689
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 9594994689
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994689
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 9594994689
[cn5:14174] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 9594994689
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 13889961985
[cn5:14174] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 13889961985
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 13889961985
[cn5:14174] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 13889961985
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 13889961985
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 13889961985
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 13889961985
[cn5:14174] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 13889961985
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 13889961985
[cn4:17832] mca: base: components_open: opening grpcomm components
[cn4:17832] mca: base: components_open: found loaded component bad
[cn4:17832] mca: base: components_open: component bad open function successful
[cn4:17832] mca:base:select: Auto-selecting grpcomm components
[cn4:17832] mca:base:select:(grpcomm) Querying component [bad]
[cn4:17832] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn4:17832] mca:base:select:(grpcomm) Selected component [bad]
[cn4:17832] [[15336,1],7] grpcomm:base:receive start comm
[cn4:17832] [[15336,1],7] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn4:17832] [[15336,1],7] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 48
[cn3:21419] [[15336,1],4] waiting for connect completion to [[15336,0],2] - activating send event
[cn3:21419] [[15336,1],4] tcp:send_handler called to send to peer [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_handler CONNECTING
[cn3:21419] [[15336,1],4]:tcp:complete_connect called for peer [[15336,0],2] on socket 11
[cn3:21419] [[15336,1],4] tcp_peer_complete_connect: sending ack to [[15336,0],2]
[cn3:21419] [[15336,1],4] SEND CONNECT ACK
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key opal.cred on proc 18184929281
[cn3:21419] dstore_hash:fetch key opal.cred for proc 18184929281 not found
[cn3:21419] [[15336,1],4] send blocking of 47 bytes to socket 11
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 156
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 156 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2748
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 2748 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 190
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 190 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 826
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 826 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 156 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 2748 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 190 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 826 bytes from [[15336,0],2] for tag 2
[cn2:01045] [[15336,1],2]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01045] [[15336,1],2]:tcp:recv:handler CONNECTED
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961985
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 13889961985
[cn5:14174] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 13889961985
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18184929281
[cn5:14174] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 18184929281
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18184929281
[cn5:14174] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 18184929281
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18184929281
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 18184929281
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 18184929281
[cn5:14174] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 18184929281
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 18184929281
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929281
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18184929281
[cn5:14174] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 18184929281
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 22479896577
[cn5:14174] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 22479896577
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 22479896577
[cn5:14174] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 22479896577
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 22479896577
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 22479896577
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 22479896577
[cn4:17832] [[15336,1],7] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn4:17832] [[15336,1],7] setting up session dir with
	tmpdir: /tmp
	host cn4
[cn3:21419] [[15336,1],4] blocking send complete to socket 11
[cn3:21419] [[15336,1],4] tcp_peer_complete_connect: setting read event on connection to [[15336,0],2]
[cn3:21419] [[15336,1],4]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21419] [[15336,1],4] RECV CONNECT ACK FROM [[15336,0],2] ON SOCKET 11
[cn3:21419] [[15336,1],4] waiting for connect ack from [[15336,0],2]
[cn3:21419] [[15336,1],4] connect ack received from [[15336,0],2]
[cn3:21419] [[15336,1],4] connect-ack recvd from [[15336,0],2]
[cn3:21419] [[15336,1],4] connect-ack header from [[15336,0],2] is okay
[cn3:21419] [[15336,1],4] waiting for connect ack from [[15336,0],2]
[cn3:21419] [[15336,1],4] connect ack received from [[15336,0],2]
[cn3:21419] [[15336,1],4] connect-ack version from [[15336,0],2] matches ours
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 482
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 482 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1447
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1447 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 160
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 160 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 831
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 831 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 482 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 1447 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 160 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 831 bytes from [[15336,0],2] for tag 2
[cn2:01045] [[15336,1],2] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 14140 BYTES FOR DEST [[15336,1],2] TAG 20
[cn2:01045] [[15336,1],2] DELIVERING TO RML
[cn2:01045] [[15336,1],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] [[15336,1],2] message received 14140 bytes from [[15336,0],1] for tag 20
[cn2:01045] [[15336,1],2] registering sync ack recvd
[cn5:14174] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 22479896577
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 22479896577
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 22479896577
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 22479896577
[cn5:14174] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 22479896577
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 26774863873
[cn5:14174] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 26774863873
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 26774863873
[cn5:14174] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 26774863873
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 26774863873
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 26774863873
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn4:17832] [[15336,1],7] posting recv
[cn4:17832] [[15336,1],7] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn3:21419] [[15336,1],4] connect-ack [[15336,0],2] authenticated
[cn3:21419] [[15336,1],4]-[[15336,0],2] tcp_peer_connected on socket 11
[cn3:21419] [[15336,1],4]-[[15336,0],2] connected: 192.168.1.3 - 192.168.1.3 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn3:21419] [[15336,1],4]:tcp:recv:handler starting send/recv events
[cn3:21419] [[15336,1],4] tcp:set_module called for peer [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_handler called to send to peer [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_handler SENDING TO [[15336,0],2]
[cn3:21419] [[15336,1],4] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 62 BYTES ON SOCKET 11
[cn3:21419] [[15336,1],4]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 608
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 608 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3280
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 3280 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 289
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 289 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 267
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 267 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 608 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 3280 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 289 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 267 bytes from [[15336,0],2] for tag 2
[cn2:01046] [[15336,1],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01046] [[15336,1],3]:tcp:recv:handler CONNECTED
[cn2:01046] [[15336,1],3] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 14140 BYTES FOR DEST [[15336,1],3] TAG 20
[cn2:01046] [[15336,1],3] DELIVERING TO RML
[cn2:01046] [[15336,1],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] message received 14140 bytes from [[15336,0],1] for tag 20
[cn2:01046] [[15336,1],3] registering sync ack recvd
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 26774863873
[cn5:14174] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 26774863873
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 26774863873
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 26774863873
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 26774863873
[cn5:14174] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 26774863873
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 31069831169
[cn5:14174] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 31069831169
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 31069831169
[cn5:14174] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 31069831169
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 31069831169
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 31069831169
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 31069831169
[cn5:14174] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 31069831169
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 31069831169
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 31069831169
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 31069831169
[cn5:14174] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 31069831169
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key opal.local.ldr[OPAL_UINT64] for proc 35364798465
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 35364798465
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 35364798465
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn4:17832] [[15336,1],7] posting recv
[cn4:17832] [[15336,1],7] posting persistent recv on tag 48 for peer [[WILDCARD],WILDCARD]
[cn4:17832] [[15336,1],7] posting recv
[cn4:17832] [[15336,1],7] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn3:21420] mca: base: components_register: registering grpcomm components
[cn3:21420] mca: base: components_register: found loaded component bad
[cn3:21420] mca: base: components_register: component bad has no register or open function
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1806
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1806 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 228
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 228 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 738
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 738 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 1806 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 228 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 738 bytes from [[15336,0],2] for tag 2
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 1005060096
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060096
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18446744069414584318
[cn2:01045] dstore:hash:store: storing key cn1[OPAL_UINT32] for proc 18446744069414584318
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 5300027392
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027392
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18446744069414584318
[cn2:01045] dstore:hash:store: storing key cn2[OPAL_UINT32] for proc 18446744069414584318
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994689
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 9594994689
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994688
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994688
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18446744069414584318
[cn2:01045] dstore:hash:store: storing key cn3[OPAL_UINT32] for proc 18446744069414584318
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 13889961984
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961984
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18446744069414584318
[cn2:01045] dstore:hash:store: storing key cn4[OPAL_UINT32] for proc 18446744069414584318
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18184929280
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929280
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18446744069414584318
[cn2:01045] dstore:hash:store: storing key cn5[OPAL_UINT32] for proc 18446744069414584318
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18446744070419644417
[cn2:01045] dstore:hash:store: storing key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18446744070419644417
[cn2:01045] dstore:hash:store: updating key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 1005060097
[cn2:01045] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 1005060097
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 1005060097
[cn2:01045] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 1005060097
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 1005060097
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 1005060097
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 1005060097
[cn2:01045] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 1005060097
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 1005060097
[cn2:01045] dstore:hash:st[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 35364798465
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 39659765761
[cn5:14174] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 39659765761
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 39659765761
[cn5:14174] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 39659765761
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 39659765761
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 39659765761
[cn5:14174] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 39659765761
[cn5:14174] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 39659765761
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 39659765761
[cn5:14174] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 39659765761
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 39659765761
[cn5:14174] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 39659765761
[cn5:14174] [[15336,1],8] Process bound at launch
[cn4:17832] procdir: /tmp/openmpi-sessions-artpol@cn4_0/15336/1/7
[cn4:17832] jobdir: /tmp/openmpi-sessions-artpol@cn4_0/15336/1
[cn4:17832] top: openmpi-sessions-artpol@cn4_0
[cn4:17832] tmp: /tmp
[cn3:21420] mca: base: components_open: opening grpcomm components
[cn3:21420] mca: base: components_open: found loaded component bad
[cn3:21420] mca: base: components_open: component bad open function successful
[cn3:21420] mca:base:select: Auto-selecting grpcomm components
[cn3:21420] mca:base:select:(grpcomm) Querying component [bad]
[cn3:21420] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn3:21420] mca:base:select:(grpcomm) Selected component [bad]
[cn3:21420] [[15336,1],5] grpcomm:base:receive start comm
[cn3:21420] [[15336,1],5] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn3:21420] [[15336,1],5] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 48
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 149
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 149 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 691
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 691 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 190
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 190 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 149 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 691 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 190 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 160
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 160 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
ore: storing key orte.hostname[OPAL_STRING] for proc 1005060097
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 1005060097
[cn2:01045] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 1005060097
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 5300027393
[cn2:01045] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 5300027393
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 5300027393
[cn2:01045] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 5300027393
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 5300027393
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 5300027393
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 5300027393
[cn2:01045] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 5300027393
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 5300027393
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027393
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 5300027393
[cn2:01045] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 5300027393
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: storing key opal.local.ldr[OPAL_UINT64] for proc 9594994689
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 9594994689
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 9594994689
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 9594994689
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 13889961985
[cn2:01045] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 13889961985
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 13889961985
[cn2:01045] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 13889961985
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 13889961985
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 13889961985
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 13889961985
[cn2:01045] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 13889961985
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn2:01045] sto[cn5:14174] MCW rank 8 bound to socket 0[core 0[hwt 0]]: [B/././.][./././.]
[cn5:14174] storing data in NONPEER dstore
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key opal.job.session.dir[OPAL_STRING] for proc 31069831169
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key opal.my.session.dir[OPAL_STRING] for proc 31069831169
[cn4:17832] [[15336,1],7] routed_binomial: init routes for proc job [15336,1]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
	daemon uri 1005060096.3;tcp://192.168.1.4:36268
[cn4:17832] [[15336,1],7] [[15336,0],3] with daemon registering
[cn3:21420] [[15336,1],5] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn3:21420] [[15336,1],5] setting up session dir with
	tmpdir: /tmp
	host cn3
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 387
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 387 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 665
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 665 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 160 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 387 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 665 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 608
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 608 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1406
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1406 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 289
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 289 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] [[15336,1],5] posting recv
[cn3:21420] [[15336,1],5] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
ring data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 13889961985
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961985
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 13889961985
[cn2:01045] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 13889961985
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18184929281
[cn2:01045] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 18184929281
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18184929281
[cn2:01045] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 18184929281
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18184929281
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 18184929281
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 18184929281
[cn2:01045] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 18184929281
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 18184929281
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929281
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18184929281
[cn2:01045] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 18184929281
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 22479896577
[cn2:01045] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 22479896577
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 22479896577
[cn2:01045] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 22479896577
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 22479896577
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 22479896577
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 22479896577
[cn2:01045] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 22479896577
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 22479896577
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 22479896577
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 22479896577
[cn2:01045] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 22479896577
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 26774863873
[cn2:01045] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 26774863873
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 26774863873
[cn2:01045] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 26774863873
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 26774863873
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:has[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: updating key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn5:14174] storing data in PEER dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 35364798465
[cn4:17832] [[15336,1],7] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 20
[cn4:17832] [[15336,1],7] rml_send_buffer to peer [[15336,0],3] at tag 1
[cn4:17832] [[15336,1],7] registering sync waiting for ack
[cn4:17832] [[15336,1],7] waiting on progress thread at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/routed/base/routed_base_fns.c:350
[cn4:17832] [[15336,1],7]: set_addr to uri 1005060096.3;tcp://192.168.1.4:36268
[cn4:17832] [[15336,1],7]:set_addr checking if peer [[15336,0],3] is reachable via component tcp
[cn4:17832] [[15336,1],7] oob:tcp: working peer [[15336,0],3] address tcp://192.168.1.4:36268
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 608 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 1406 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 289 bytes from [[15336,0],2] for tag 2
h:store storing data for proc 26774863873
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 26774863873
[cn2:01045] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 26774863873
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 26774863873
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 26774863873
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 26774863873
[cn2:01045] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 26774863873
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 31069831169
[cn2:01045] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 31069831169
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 31069831169
[cn2:01045] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 31069831169
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 31069831169
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 31069831169
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 31069831169
[cn2:01045] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 31069831169
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 31069831169
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 31069831169
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 31069831169
[cn2:01045] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 31069831169
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 35364798465
[cn2:01045] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 35364798465
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 35364798465
[cn2:01045] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 35364798465
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 35364798465
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 35364798465
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 35364798465
[cn2:01045] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 35364798465
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 35364798465
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 35364798465
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 35364798465
[cn2:01045] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 35364798465
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 39659765761
[cn2:01045] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for [cn5:14175] [[15336,1],9]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14175] [[15336,1],9]:tcp:recv:handler CONNECTED
[cn5:14175] [[15336,1],9] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 14140 BYTES FOR DEST [[15336,1],9] TAG 20
[cn5:14175] [[15336,1],9] DELIVERING TO RML
[cn5:14175] [[15336,1],9] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14175] [[15336,1],9] message received 14140 bytes from [[15336,0],4] for tag 20
[cn5:14175] [[15336,1],9] registering sync ack recvd
[cn4:17832] [[15336,1],7] PEER [[15336,0],3] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn4:17832] [[15336,1],7] PASSING ADDR 192.168.1.4 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn4:17832] [[15336,1],7]:tcp set addr for peer [[15336,0],3]
[cn4:17832] [[15336,1],7]: peer [[15336,0],3] is reachable via component tcp
[cn4:17832] [[15336,1],7] posting recv
[cn4:17832] [[15336,1],7] posting non-persistent recv on tag 20 for peer [[WILDCARD],WILDCARD]
[cn4:17832] [[15336,1],7] rml_send_msg to peer [[15336,0],3] at tag 1
[cn4:17832] [[15336,1],7] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17832] [[15336,1],7]:tcp:processing set_peer cmd for interface eth0
[cn4:17832] [[15336,1],7] SET_PEER ADDING PEER [[15336,0],3]
[cn4:17832] [[15336,1],7] set_peer: peer [[15336,0],3] is listening on interface eth0 net 192.168.1.4 port 36268
[cn4:17832] [[15336,1],7] oob:base:send to target [[15336,0],3]
[cn4:17832] [[15336,1],7] oob:tcp:send_nb to peer [[15336,0],3]:1
[cn4:17832] [[15336,1],7] tcp:send_nb to peer [[15336,0],3]
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:1
[cn3:21420] [[15336,1],5] posting recv
[cn3:21420] [[15336,1],5] posting persistent recv on tag 48 for peer [[WILDCARD],WILDCARD]
[cn3:21420] [[15336,1],5] posting recv
[cn3:21420] [[15336,1],5] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1893
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1893 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 908
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 908 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 206
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 206 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1893 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 908 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 206 bytes from [[15336,0],2] for tag 2
proc 39659765761
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 39659765761
[cn2:01045] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 39659765761
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 39659765761
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 39659765761
[cn2:01045] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 39659765761
[cn2:01045] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 39659765761
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 39659765761
[cn2:01045] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 39659765761
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 39659765761
[cn2:01045] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 39659765761
[cn2:01045] [[15336,1],2] Process bound at launch
[cn2:01045] MCW rank 2 bound to socket 0[core 0[hwt 0]]: [B/././.][./././.]
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: updating key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn2:01045] storing data in PEER dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn2:01045] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 9594994689
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 1005060096
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060096
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18446744069414584318
[cn5:14175] dstore:hash:store: storing key cn1[OPAL_UINT32] for proc 18446744069414584318
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 5300027392
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027392
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18446744069414584318
[cn5:14175] dstore:hash:store: storing key cn2[OPAL_UINT32] for proc 18446744069414584318
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 9594994688
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994688
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18446744069414584318
[cn5:14175] dstore:hash:store: storing key cn3[OPAL_UINT32] for proc 18446744069414584318
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 13889961984
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961984
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18446744069414584318
[cn5:14175] dstore:hash:store: storing key cn4[OPAL_UINT32] for proc 18446744069414584318
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18184929280
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929280
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18446744069414584318
[cn5:14175] dstore:hash:store: storing key cn5[OPAL_UINT32] for proc 18446744069414584318
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 39659765761
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 39659765761
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18446744070419644417
[cn5:14175] dstore:hash:store: storing key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18446744070419644417
[cn5:14175] dstore:hash:store: updating key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 1005060097
[cn5:14175] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 1005060097
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 1005060097
[cn5:14175] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 1005060097
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 1005060097
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 1005060097
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 1005060097
[cn5:14175] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 1005060097
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 1005060097
[cn5:14175] dstore:has[cn4:17832] [[15336,1],7] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_nb: initiating connection to [[15336,0],3]
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],3]
[cn4:17832] [[15336,1],7] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],3] via interface eth0
[cn4:17832] [[15336,1],7] oob:tcp:peer creating socket to [[15336,0],3]
[cn4:17832] [[15336,1],7] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],3] via interface eth0 on socket 11
[cn4:17832] [[15336,1],7] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],3] on 192.168.1.4:36268 - 0 retries
[cn3:21420] procdir: /tmp/openmpi-sessions-artpol@cn3_0/15336/1/5
[cn3:21420] jobdir: /tmp/openmpi-sessions-artpol@cn3_0/15336/1
[cn3:21420] top: openmpi-sessions-artpol@cn3_0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 227
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 227 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 464
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 464 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 805
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 805 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 713
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 713 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 227 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 464 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 805 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 713 bytes from [[15336,0],2] for tag 2
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 1005060096
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060096
[cn5:14174] mca: base: components_register: registering bml components
[cn5:14174] mca: base: components_register: found loaded component r2
[cn5:14174] mca: base: components_register: component r2 register function successful
[cn5:14174] mca: base: components_open: opening bml components
[cn5:14174] mca: base: components_open: found loaded component r2
[cn5:14174] mca: base: components_open: component r2 open function successful
[cn4:17832] [[15336,1],7] waiting for connect completion to [[15336,0],3] - activating send event
[cn4:17832] [[15336,1],7] tcp:send_handler called to send to peer [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_handler CONNECTING
[cn4:17832] [[15336,1],7]:tcp:complete_connect called for peer [[15336,0],3] on socket 11
[cn4:17832] [[15336,1],7] tcp_peer_complete_connect: sending ack to [[15336,0],3]
[cn4:17832] [[15336,1],7] SEND CONNECT ACK
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key opal.cred on proc 31069831169
[cn4:17832] dstore_hash:fetch key opal.cred for proc 31069831169 not found
[cn4:17832] [[15336,1],7] send blocking of 47 bytes to socket 11
[cn4:17832] [[15336,1],7] blocking send complete to socket 11
[cn3:21420] tmp: /tmp
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key opal.job.session.dir[OPAL_STRING] for proc 22479896577
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key opal.my.session.dir[OPAL_STRING] for proc 22479896577
[cn3:21420] [[15336,1],5] routed_binomial: init routes for proc job [15336,1]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
	daemon uri 1005060096.2;tcp://192.168.1.3:42008
[cn3:21420] [[15336,1],5] [[15336,0],2] with daemon registering
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1643
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1643 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 133
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 133 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 665
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 665 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1643 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 133 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 665 bytes from [[15336,0],2] for tag 2
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18446744069414584318
[cn2:01046] dstore:hash:store: storing key cn1[OPAL_UINT32] for proc 18446744069414584318
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 5300027392
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027392
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18446744069414584318
[cn2:01046] dstore:hash:store: storing key cn2[OPAL_UINT32] for proc 18446744069414584318
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961985
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 13889961985
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 9594994688
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994688
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18446744069414584318
[cn2:01046] dstore:hash:store: storing key cn3[OPAL_UINT32] for proc 18446744069414584318
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961984
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961984
h:store: storing key orte.hostname[OPAL_STRING] for proc 1005060097
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 1005060097
[cn5:14175] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 1005060097
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 5300027393
[cn5:14175] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 5300027393
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 5300027393
[cn5:14175] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 5300027393
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 5300027393
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 5300027393
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 5300027393
[cn5:14175] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 5300027393
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 5300027393
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027393
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 5300027393
[cn5:14175] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 5300027393
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 9594994689
[cn5:14175] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 9594994689
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 9594994689
[cn5:14175] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 9594994689
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 9594994689
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 9594994689
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 9594994689
[cn5:14175] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 9594994689
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 9594994689
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994689
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 9594994689
[cn5:14175] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 9594994689
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 13889961985
[cn5:14175] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 13889961985
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 13889961985
[cn5:14175] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 13889961985
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 13889961985
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 13889961985
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn5:14175] st[cn4:17832] [[15336,1],7] tcp_peer_complete_connect: setting read event on connection to [[15336,0],3]
[cn3:21420] [[15336,1],5] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 20
[cn3:21420] [[15336,1],5] rml_send_buffer to peer [[15336,0],2] at tag 1
[cn3:21420] [[15336,1],5] registering sync waiting for ack
[cn3:21420] [[15336,1],5] waiting on progress thread at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/routed/base/routed_base_fns.c:350
[cn3:21420] [[15336,1],5]: set_addr to uri 1005060096.2;tcp://192.168.1.3:42008
[cn3:21420] [[15336,1],5]:set_addr checking if peer [[15336,0],2] is reachable via component tcp
[cn3:21420] [[15336,1],5] oob:tcp: working peer [[15336,0],2] address tcp://192.168.1.3:42008
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1772
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1772 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1217
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1217 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 684
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 684 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1772 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 1217 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 684 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2352
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2352 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 275
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 275 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1183
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1183 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18446744069414584318
[cn2:01046] dstore:hash:store: storing key cn4[OPAL_UINT32] for proc 18446744069414584318
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18184929280
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929280
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18446744069414584318
[cn2:01046] dstore:hash:store: storing key cn5[OPAL_UINT32] for proc 18446744069414584318
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18446744070419644417
[cn2:01046] dstore:hash:store: storing key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18446744070419644417
[cn2:01046] dstore:hash:store: updating key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 1005060097
[cn2:01046] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 1005060097
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 1005060097
[cn2:01046] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 1005060097
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 1005060097
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 1005060097
oring data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 13889961985
[cn5:14175] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 13889961985
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 13889961985
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961985
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 13889961985
[cn5:14175] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 13889961985
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18184929281
[cn5:14175] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 18184929281
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18184929281
[cn5:14175] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 18184929281
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18184929281
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 18184929281
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 18184929281
[cn5:14175] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 18184929281
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 18184929281
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929281
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18184929281
[cn5:14175] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 18184929281
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 22479896577
[cn5:14175] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 22479896577
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 22479896577
[cn5:14175] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 22479896577
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 22479896577
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 22479896577
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 22479896577
[cn5:14175] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 22479896577
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 22479896577
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 22479896577
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 22479896577
[cn5:14175] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 22479896577
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 26774863873
[cn5:14175] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 26774863873
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 26774863873
[cn5:14175] dstore:hash[cn4:17832] [[15336,1],7]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17832] [[15336,1],7] RECV CONNECT ACK FROM [[15336,0],3] ON SOCKET 11
[cn4:17832] [[15336,1],7] waiting for connect ack from [[15336,0],3]
[cn4:17832] [[15336,1],7] connect ack received from [[15336,0],3]
[cn4:17832] [[15336,1],7] connect-ack recvd from [[15336,0],3]
[cn4:17832] [[15336,1],7] connect-ack header from [[15336,0],3] is okay
[cn4:17832] [[15336,1],7] waiting for connect ack from [[15336,0],3]
[cn4:17832] [[15336,1],7] connect ack received from [[15336,0],3]
[cn4:17832] [[15336,1],7] connect-ack version from [[15336,0],3] matches ours
[cn4:17832] [[15336,1],7] connect-ack [[15336,0],3] authenticated
[cn4:17832] [[15336,1],7]-[[15336,0],3] tcp_peer_connected on socket 11
[cn4:17832] [[15336,1],7]-[[15336,0],3] connected: 192.168.1.4 - 192.168.1.4 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn4:17832] [[15336,1],7]:tcp:recv:handler starting send/recv events
[cn4:17832] [[15336,1],7] tcp:set_module called for peer [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_handler called to send to peer [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_handler SENDING TO [[15336,0],3]
[cn3:21420] [[15336,1],5] PEER [[15336,0],2] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn3:21420] [[15336,1],5] PASSING ADDR 192.168.1.3 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn3:21420] [[15336,1],5]:tcp set addr for peer [[15336,0],2]
[cn3:21420] [[15336,1],5]: peer [[15336,0],2] is reachable via component tcp
[cn3:21420] [[15336,1],5] posting recv
[cn3:21420] [[15336,1],5] posting non-persistent recv on tag 20 for peer [[WILDCARD],WILDCARD]
[cn3:21420] [[15336,1],5] rml_send_msg to peer [[15336,0],2] at tag 1
[cn3:21420] [[15336,1],5] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 2352 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 275 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 1183 bytes from [[15336,0],2] for tag 2
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 1005060097
[cn2:01046] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 1005060097
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 1005060097
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060097
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 1005060097
[cn2:01046] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 1005060097
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 5300027393
[cn2:01046] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 5300027393
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 5300027393
[cn2:01046] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 5300027393
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 5300027393
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 5300027393
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 5300027393
[cn2:01046] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 5300027393
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 5300027393
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027393
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 5300027393
[cn2:01046] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 5300027393
:store: storing key orte.node.rank[OPAL_UINT16] for proc 26774863873
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 26774863873
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 26774863873
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 26774863873
[cn5:14175] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 26774863873
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 26774863873
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 26774863873
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 26774863873
[cn5:14175] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 26774863873
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 31069831169
[cn5:14175] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 31069831169
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 31069831169
[cn5:14175] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 31069831169
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 31069831169
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 31069831169
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 31069831169
[cn5:14175] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 31069831169
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 31069831169
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 31069831169
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 31069831169
[cn5:14175] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 31069831169
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: storing key opal.local.ldr[OPAL_UINT64] for proc 39659765761
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 35364798465
[cn5:14175] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 35364798465
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 35364798465
[cn5:14175] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 35364798465
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 35364798465
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 35364798465
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 35364798465
[cn5:14175] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 35364798465
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn5:14175] storing data in NONPEER dstore
[cn5:14175[cn4:17832] [[15336,1],7] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 62 BYTES ON SOCKET 11
[cn4:17832] [[15336,1],7]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21420] [[15336,1],5]:tcp:processing set_peer cmd for interface eth0
[cn3:21420] [[15336,1],5] SET_PEER ADDING PEER [[15336,0],2]
[cn3:21420] [[15336,1],5] set_peer: peer [[15336,0],2] is listening on interface eth0 net 192.168.1.3 port 42008
[cn3:21420] [[15336,1],5] oob:base:send to target [[15336,0],2]
[cn3:21420] [[15336,1],5] oob:tcp:send_nb to peer [[15336,0],2]:1
[cn3:21420] [[15336,1],5] tcp:send_nb to peer [[15336,0],2]
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:1
[cn3:21420] [[15336,1],5] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_nb: initiating connection to [[15336,0],2]
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2445
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2445 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1898
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1898 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 351
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 351 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 575
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 575 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 2445 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 1898 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 351 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 575 bytes from [[15336,0],2] for tag 2
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: storing key opal.local.ldr[OPAL_UINT64] for proc 13889961985
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 9594994689
[cn2:01046] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 9594994689
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 9594994689
[cn2:01046] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 9594994689
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 9594994689
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 9594994689
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 9594994689
[cn2:01046] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 9594994689
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 9594994689
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994689
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 9594994689
[cn2:01046] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 9594994689
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 13889961985
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 13889961985
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
] dstore:hash:store storing data for proc 35364798465
[cn5:14175] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 35364798465
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 35364798465
[cn5:14175] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 35364798465
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 39659765761
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 39659765761
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 39659765761
[cn5:14175] [[15336,1],9] Process bound at launch
[cn5:14175] MCW rank 9 bound to socket 0[core 1[hwt 0]]: [./B/./.][./././.]
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: updating key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn5:14175] storing data in PEER dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 39659765761
[cn4:17831] [[15336,1],6]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17831] [[15336,1],6]:tcp:recv:handler CONNECTED
[cn4:17831] [[15336,1],6]:tcp:recv:handler allocate new recv msg
[cn4:17831] [[15336,1],6]:tcp:recv:handler read hdr
[cn4:17831] [[15336,1],6]:tcp:recv:handler allocate data region of size 14140
[cn3:21420] [[15336,1],5] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],2] via interface eth0
[cn3:21420] [[15336,1],5] oob:tcp:peer creating socket to [[15336,0],2]
[cn3:21420] [[15336,1],5] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],2] via interface eth0 on socket 11
[cn3:21420] [[15336,1],5] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],2] on 192.168.1.3:42008 - 0 retries
[cn3:21420] [[15336,1],5] waiting for connect completion to [[15336,0],2] - activating send event
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2289
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2289 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 171
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 171 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 810
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 810 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 2289 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 171 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],3] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] routed_binomial: init routes for HNP job [15336,1]
[cn1:14209] [[15336,0],0] routed_binomial:callback got uri 1005060097.6;tcp://192.168.1.4:49793 for job [15336,1] rank 6
[cn1:14209] [[15336,0],0] routed_binomial:callback got uri 1005060097.7;tcp://192.168.1.4:41093 for job [15336,1] rank 7
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn1:14209] [[15336,0],0] message received 810 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2943
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2943 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 323
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 323 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 351
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 351 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1217
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1217 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 13889961985
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18184929281
[cn2:01046] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 18184929281
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18184929281
[cn2:01046] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 18184929281
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18184929281
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 18184929281
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 18184929281
[cn2:01046] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 18184929281
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 18184929281
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929281
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18184929281
[cn2:01046] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 18184929281
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 22479896577
[cn2:01046] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 22479896577
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 22479896577
[cn2:01046] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 22479896577
[cn2:01046] storing data in INTERNAL dstore
[cn5:14175] mca: base: components_register: registering bml components
[cn5:14175] mca: base: components_register: found loaded component r2
[cn3:21420] [[15336,1],5] tcp:send_handler called to send to peer [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_handler CONNECTING
[cn3:21420] [[15336,1],5]:tcp:complete_connect called for peer [[15336,0],2] on socket 11
[cn3:21420] [[15336,1],5] tcp_peer_complete_connect: sending ack to [[15336,0],2]
[cn3:21420] [[15336,1],5] SEND CONNECT ACK
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key opal.cred on proc 22479896577
[cn3:21420] dstore_hash:fetch key opal.cred for proc 22479896577 not found
[cn3:21420] [[15336,1],5] send blocking of 47 bytes to socket 11
[cn3:21420] [[15336,1],5] blocking send complete to socket 11
[cn3:21420] [[15336,1],5] tcp_peer_complete_connect: setting read event on connection to [[15336,0],2]
[cn1:14209] [[15336,0],0] message received 2943 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 323 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 351 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 1217 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1844
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1844 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 788
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 788 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 156
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 156 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 275
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 275 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] dstore:hash:store storing data for proc 22479896577
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 22479896577
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 22479896577
[cn2:01046] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 22479896577
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 22479896577
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 22479896577
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 22479896577
[cn2:01046] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 22479896577
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 26774863873
[cn2:01046] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 26774863873
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 26774863873
[cn2:01046] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 26774863873
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 26774863873
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 26774863873
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 26774863873
[cn2:01046] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 26774863873
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 26774863873
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 26774863873
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 26774863873
[cn2:01046] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 26774863873
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 31069831169
[cn2:01046] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 31069831169
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 31069831169
[cn5:14175] mca: base: components_register: component r2 register function successful
[cn5:14175] mca: base: components_open: opening bml components
[cn5:14175] mca: base: components_open: found loaded component r2
[cn5:14175] mca: base: components_open: component r2 open function successful
[cn4:17832] [[15336,1],7]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17832] [[15336,1],7]:tcp:recv:handler CONNECTED
[cn4:17832] [[15336,1],7]:tcp:recv:handler allocate new recv msg
[cn4:17832] [[15336,1],7]:tcp:recv:handler read hdr
[cn4:17832] [[15336,1],7]:tcp:recv:handler allocate data region of size 14140
[cn3:21420] [[15336,1],5]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21420] [[15336,1],5] RECV CONNECT ACK FROM [[15336,0],2] ON SOCKET 11
[cn3:21420] [[15336,1],5] waiting for connect ack from [[15336,0],2]
[cn3:21420] [[15336,1],5] connect ack received from [[15336,0],2]
[cn3:21420] [[15336,1],5] connect-ack recvd from [[15336,0],2]
[cn3:21420] [[15336,1],5] connect-ack header from [[15336,0],2] is okay
[cn3:21420] [[15336,1],5] waiting for connect ack from [[15336,0],2]
[cn3:21420] [[15336,1],5] connect ack received from [[15336,0],2]
[cn3:21420] [[15336,1],5] connect-ack version from [[15336,0],2] matches ours
[cn3:21420] [[15336,1],5] connect-ack [[15336,0],2] authenticated
[cn3:21420] [[15336,1],5]-[[15336,0],2] tcp_peer_connected on socket 11
[cn3:21420] [[15336,1],5]-[[15336,0],2] connected: 192.168.1.3 - 192.168.1.3 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn3:21420] [[15336,1],5]:tcp:recv:handler starting send/recv events
[cn3:21420] [[15336,1],5] tcp:set_module called for peer [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_handler called to send to peer [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_handler SENDING TO [[15336,0],2]
[cn1:14209] [[15336,0],0] message received 1844 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 788 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 156 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 275 bytes from [[15336,0],2] for tag 2
[cn2:01046] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 31069831169
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 31069831169
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 31069831169
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 31069831169
[cn2:01046] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 31069831169
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 31069831169
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 31069831169
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 31069831169
[cn2:01046] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 31069831169
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 35364798465
[cn2:01046] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 35364798465
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 35364798465
[cn2:01046] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 35364798465
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 35364798465
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn5:14174] mca: base: components_register: registering btl components
[cn5:14174] mca: base: components_register: found loaded component openib
[cn5:14174] mca: base: components_register: component openib register function successful
[cn5:14174] mca: base: components_register: found loaded component self
[cn5:14174] mca: base: components_register: component self register function successful
[cn5:14174] mca: base: components_open: opening btl components
[cn5:14174] mca: base: components_open: found loaded component openib
[cn5:14174] mca: base: components_open: component openib open function successful
[cn5:14174] mca: base: components_open: found loaded component self
[cn5:14174] mca: base: components_open: component self open function successful
[cn4:17831] [[15336,1],6]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17831] [[15336,1],6]:tcp:recv:handler CONNECTED
[cn3:21420] [[15336,1],5] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 62 BYTES ON SOCKET 11
[cn3:21420] [[15336,1],5]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2324
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2324 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 175
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 175 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 429
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 429 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 273
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 273 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 2324 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 175 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 429 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 273 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 643
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 643 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 83
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 83 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 108
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 108 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 35364798465
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 35364798465
[cn2:01046] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 35364798465
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 35364798465
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 35364798465
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 35364798465
[cn2:01046] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 35364798465
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 39659765761
[cn2:01046] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 39659765761
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 39659765761
[cn2:01046] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 39659765761
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 39659765761
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 39659765761
[cn2:01046] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 39659765761
[cn2:01046] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 39659765761
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 39659765761
[cn2:01046] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 39659765761
[cn2:01046] storing data in INTERNAL dstore
[cn5:14175] mca: base: components_register: registering btl components
[cn5:14175] mca: base: components_register: found loaded component openib
[cn4:17831] [[15336,1],6] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 14140 BYTES FOR DEST [[15336,1],6] TAG 20
[cn4:17831] [[15336,1],6] DELIVERING TO RML
[cn4:17831] [[15336,1],6] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] [[15336,1],6] message received 14140 bytes from [[15336,0],3] for tag 20
[cn3:21419] [[15336,1],4]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21419] [[15336,1],4]:tcp:recv:handler CONNECTED
[cn3:21419] [[15336,1],4]:tcp:recv:handler allocate new recv msg
[cn3:21419] [[15336,1],4]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 238
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 238 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 643 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 83 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 108 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 238 bytes from [[15336,0],1] for tag 2
[cn5:14175] mca: base: components_register: component openib register function successful
[cn5:14175] mca: base: components_register: found loaded component self
[cn5:14175] mca: base: components_register: component self register function successful
[cn5:14175] mca: base: components_open: opening btl components
[cn5:14175] mca: base: components_open: found loaded component openib
[cn5:14175] mca: base: components_open: component openib open function successful
[cn5:14175] mca: base: components_open: found loaded component self
[cn5:14175] mca: base: components_open: component self open function successful
[cn4:17831] [[15336,1],6] registering sync ack recvd
[cn3:21419] [[15336,1],4]:tcp:recv:handler allocate data region of size 14140
[cn2:01046] dstore:hash:store storing data for proc 39659765761
[cn2:01046] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 39659765761
[cn2:01046] [[15336,1],3] Process bound at launch
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 106
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 106 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 170
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 170 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 555
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 555 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 106 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 170 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 555 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] routed_binomial: init routes for HNP job [15336,1]
[cn1:14209] [[15336,0],0] routed_binomial:callback got uri 1005060097.4;tcp://192.168.1.3:50297 for job [15336,1] rank 4
[cn1:14209] [[15336,0],0] routed_binomial:callback got uri 1005060097.5;tcp://192.168.1.3:37735 for job [15336,1] rank 5
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn2:01046] MCW rank 3 bound to socket 0[core 1[hwt 0]]: [./B/./.][./././.]
[cn5:14174] mca: base: components_register: registering pml components
[cn5:14174] mca: base: components_register: found loaded component v
[cn4:17832] [[15336,1],7]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17832] [[15336,1],7]:tcp:recv:handler CONNECTED
[cn4:17832] [[15336,1],7] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 14140 BYTES FOR DEST [[15336,1],7] TAG 20
[cn4:17832] [[15336,1],7] DELIVERING TO RML
[cn4:17832] [[15336,1],7] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17832] [[15336,1],7] message received 14140 bytes from [[15336,0],3] for tag 20
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 430
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 430 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 83
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 83 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 273
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 273 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 430 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 83 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 273 bytes from [[15336,0],2] for tag 2
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: updating key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn2:01046] storing data in PEER dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 13889961985
[cn5:14174] mca: base: components_register: component v register function successful
[cn5:14174] mca: base: components_register: found loaded component bfo
[cn4:17832] [[15336,1],7] registering sync ack recvd
[cn3:21420] [[15336,1],5]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21420] [[15336,1],5]:tcp:recv:handler CONNECTED
[cn3:21420] [[15336,1],5]:tcp:recv:handler allocate new recv msg
[cn3:21420] [[15336,1],5]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 464
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 464 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 117
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 117 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 882
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 882 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 108
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 108 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 464 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 117 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 882 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 108 bytes from [[15336,0],2] for tag 2
[cn2:01046] mca: base: components_register: registering bml components
[cn2:01046] mca: base: components_register: found loaded component r2
[cn2:01046] mca: base: components_register: component r2 register function successful
[cn2:01046] mca: base: components_open: opening bml components
[cn2:01046] mca: base: components_open: found loaded component r2
[cn2:01046] mca: base: components_open: component r2 open function successful
[cn5:14174] mca: base: components_register: component bfo register function successful
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 1005060096
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060096
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18446744069414584318
[cn4:17831] dstore:hash:store: storing key cn1[OPAL_UINT32] for proc 18446744069414584318
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 5300027392
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027392
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18446744069414584318
[cn4:17831] dstore:hash:store: storing key cn2[OPAL_UINT32] for proc 18446744069414584318
[cn4:17831] storing data in INTERNAL dstore
[cn3:21420] [[15336,1],5]:tcp:recv:handler allocate data region of size 14140
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 464
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 464 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1601
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1601 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 555
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 555 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 464 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 1601 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 555 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 788
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 788 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 83
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 83 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] mca: base: components_register: registering bml components
[cn2:01045] mca: base: components_register: found loaded component r2
[cn2:01045] mca: base: components_register: component r2 register function successful
[cn2:01045] mca: base: components_open: opening bml components
[cn2:01045] mca: base: components_open: found loaded component r2
[cn2:01045] mca: base: components_open: component r2 open function successful
[cn5:14174] mca: base: components_register: found loaded component cm
[cn5:14174] mca: base: components_register: component cm register function successful
[cn4:17831] dstore:hash:store storing data for proc 9594994688
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994688
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18446744069414584318
[cn4:17831] dstore:hash:store: storing key cn3[OPAL_UINT32] for proc 18446744069414584318
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 13889961984
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961984
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18446744069414584318
[cn4:17831] dstore:hash:store: storing key cn4[OPAL_UINT32] for proc 18446744069414584318
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 26774863873
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 26774863873
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18184929280
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929280
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18446744069414584318
[cn4:17831] dstore:hash:store: storing key cn5[OPAL_UINT32] for proc 18446744069414584318
[cn3:21419] [[15336,1],4]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21419] [[15336,1],4]:tcp:recv:handler CONNECTED
[cn3:21419] [[15336,1],4] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 14140 BYTES FOR DEST [[15336,1],4] TAG 20
[cn3:21419] [[15336,1],4] DELIVERING TO RML
[cn3:21419] [[15336,1],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] [[15336,1],4] message received 14140 bytes from [[15336,0],2] for tag 20
[cn1:14209] [[15336,0],0] message received 788 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 83 bytes from [[15336,0],2] for tag 2
[cn2:01045] mca: base: components_register: registering btl components
[cn2:01045] mca: base: components_register: found loaded component openib
[cn2:01045] mca: base: components_register: component openib register function successful
[cn2:01045] mca: base: components_register: found loaded component self
[cn2:01045] mca: base: components_register: component self register function successful
[cn2:01045] mca: base: components_open: opening btl components
[cn2:01045] mca: base: components_open: found loaded component openib
[cn2:01045] mca: base: components_open: component openib open function successful
[cn2:01045] mca: base: components_open: found loaded component self
[cn2:01045] mca: base: components_open: component self open function successful
[cn5:14174] mca: base: components_register: found loaded component ob1
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18446744070419644417
[cn4:17831] dstore:hash:store: storing key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18446744070419644417
[cn4:17831] dstore:hash:store: updating key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 1005060097
[cn4:17831] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 1005060097
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 1005060097
[cn4:17831] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 1005060097
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 1005060097
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 1005060097
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 1005060097
[cn4:17831] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 1005060097
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 1005060097
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060097
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 1005060097
[cn4:17831] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 1005060097
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 5300027393
[cn4:17831] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 5300027393
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 5300027393
[cn4:17831] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 5300027393
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 5300027393
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 5300027393
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 5300027393
[cn4:17831] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 5300027393
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 5300027393
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027393
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 5300027393
[cn4:17831] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 5300027393
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 9594994689
[cn4:17831] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 9594994689
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 9594994689
[cn4:17831] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 9594994689
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 9594994689
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn4:17831] storing data in NONPEER dstore
[cn4:17831[cn3:21419] [[15336,1],4] registering sync ack recvd
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 788
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 788 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 245
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 245 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 156
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 156 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 788 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 245 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 156 bytes from [[15336,0],2] for tag 2
[cn2:01046] mca: base: components_register: registering btl components
[cn2:01046] mca: base: components_register: found loaded component openib
[cn2:01046] mca: base: components_register: component openib register function successful
[cn2:01046] mca: base: components_register: found loaded component self
[cn2:01046] mca: base: components_register: component self register function successful
[cn2:01046] mca: base: components_open: opening btl components
[cn2:01046] mca: base: components_open: found loaded component openib
[cn2:01046] mca: base: components_open: component openib open function successful
[cn2:01046] mca: base: components_open: found loaded component self
[cn2:01046] mca: base: components_open: component self open function successful
[cn5:14174] mca: base: components_register: component ob1 register function successful
[cn5:14174] mca: base: components_open: opening pml components
[cn5:14174] mca: base: components_open: found loaded component v
] dstore:hash:store storing data for proc 9594994689
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 9594994689
[cn4:17831] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 9594994689
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 9594994689
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994689
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 9594994689
[cn4:17831] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 9594994689
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 13889961985
[cn4:17831] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 13889961985
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 13889961985
[cn4:17831] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 13889961985
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 13889961985
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 13889961985
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 13889961985
[cn4:17831] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 13889961985
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 13889961985
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961985
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 13889961985
[cn4:17831] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 13889961985
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18184929281
[cn4:17831] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 18184929281
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18184929281
[cn4:17831] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 18184929281
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18184929281
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 18184929281
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 18184929281
[cn4:17831] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 18184929281
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 18184929281
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929281
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18184929281
[cn4:17831] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 18184929281
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 22479896577
[cn4:17831] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for[cn3:21420] [[15336,1],5]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21420] [[15336,1],5]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 855
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 855 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 855
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 855 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 429
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 429 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 855 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 855 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 429 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 855
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 855 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 319
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 319 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3571
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 3571 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 83
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 83 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] mca: base: components_register: registering pml components
[cn2:01045] mca: base: components_register: found loaded component v
[cn2:01045] mca: base: components_register: component v register function successful
[cn2:01045] mca: base: components_register: found loaded component bfo
[cn2:01045] mca: base: components_register: component bfo register function successful
[cn2:01045] mca: base: components_register: found loaded component cm
[cn2:01045] mca: base: components_register: component cm register function successful
[cn2:01045] mca: base: components_register: found loaded component ob1
[cn2:01045] mca: base: components_register: component ob1 register function successful
[cn2:01045] mca: base: components_open: opening pml components
[cn2:01045] mca: base: components_open: found loaded component v
[cn5:14175] mca: base: components_register: registering pml components
[cn5:14175] mca: base: components_register: found loaded component v
[cn5:14175] mca: base: components_register: component v register function successful
[cn5:14175] mca: base: components_register: found loaded component bfo
[cn5:14175] mca: base: components_register: component bfo register function successful
[cn5:14175] mca: base: components_register: found loaded component cm
[cn5:14175] mca: base: components_register: component cm register function successful
[cn5:14175] mca: base: components_register: found loaded component ob1
[cn5:14175] mca: base: components_register: component ob1 register function successful
[cn5:14175] mca: base: components_open: opening pml components
[cn5:14175] mca: base: components_open: found loaded component v
 proc 22479896577
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 22479896577
[cn4:17831] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 22479896577
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 22479896577
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 22479896577
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 22479896577
[cn4:17831] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 22479896577
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 22479896577
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 22479896577
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 22479896577
[cn4:17831] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 22479896577
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key opal.local.ldr[OPAL_UINT64] for proc 26774863873
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 26774863873
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 26774863873
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 26774863873
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 31069831169
[cn4:17831] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 31069831169
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 31069831169
[cn4:17831] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 31069831169
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 31069831169
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 31069831169
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 31069831169
[cn4:17831] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 31069831169
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 31069831169
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 31069831169
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 31069831169
[cn4:17831] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 31069831169
[cn4:17831] storing data in INTERNA[cn3:21420] [[15336,1],5] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 14140 BYTES FOR DEST [[15336,1],5] TAG 20
[cn3:21420] [[15336,1],5] DELIVERING TO RML
[cn3:21420] [[15336,1],5] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] [[15336,1],5] message received 14140 bytes from [[15336,0],2] for tag 20
[cn1:14209] [[15336,0],0] message received 855 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 319 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 3571 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 83 bytes from [[15336,0],2] for tag 2
[cn2:01046] mca: base: components_register: registering pml components
[cn2:01046] mca: base: components_register: found loaded component v
[cn2:01046] mca: base: components_register: component v register function successful
[cn2:01046] mca: base: components_register: found loaded component bfo
[cn2:01046] mca: base: components_register: component bfo register function successful
[cn2:01046] mca: base: components_register: found loaded component cm
[cn2:01046] mca: base: components_register: component cm register function successful
[cn2:01046] mca: base: components_register: found loaded component ob1
[cn2:01046] mca: base: components_register: component ob1 register function successful
[cn2:01046] mca: base: components_open: opening pml components
[cn2:01046] mca: base: components_open: found loaded component v
[cn5:14174] mca: base: components_open: component v open function successful
[cn5:14174] mca: base: components_open: found loaded component bfo
[cn5:14174] mca: base: components_open: component bfo open function successful
[cn5:14174] mca: base: components_open: found loaded component cm
L dstore
[cn4:17831] dstore:hash:store storing data for proc 35364798465
[cn4:17831] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 35364798465
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 35364798465
[cn4:17831] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 35364798465
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 35364798465
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 35364798465
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 35364798465
[cn4:17831] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 35364798465
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 35364798465
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 35364798465
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 35364798465
[cn4:17831] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 35364798465
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 39659765761
[cn4:17831] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 39659765761
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 39659765761
[cn4:17831] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 39659765761
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 39659765761
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 39659765761
[cn4:17831] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 39659765761
[cn4:17831] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 39659765761
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 39659765761
[cn4:17831] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 39659765761
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 39659765761
[cn4:17831] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 39659765761
[cn4:17831] [[15336,1],6] Process bound at launch
[cn4:17831] MCW rank 6 bound to socket 0[core 0[hwt 0]]: [B/././.][./././.]
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: updating key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn4:17831] storing data in PEER dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 26774863873
[cn3:21420] [[15336,1],5] registering sync ack recvd
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 319
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 319 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 319
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 319 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 882
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 882 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 319 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 319 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 882 bytes from [[15336,0],2] for tag 2
[cn2:01046] mca: base: components_open: component v open function successful
[cn2:01046] mca: base: components_open: found loaded component bfo
[cn2:01046] mca: base: components_open: component bfo open function successful
[cn2:01046] mca: base: components_open: found loaded component cm
[cn5:14175] mca: base: components_open: component v open function successful
[cn5:14175] mca: base: components_open: found loaded component bfo
[cn5:14175] mca: base: components_open: component bfo open function successful
[cn5:14175] mca: base: components_open: found loaded component cm
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 1005060096
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060096
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18446744069414584318
[cn4:17832] dstore:hash:store: storing key cn1[OPAL_UINT32] for proc 18446744069414584318
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 5300027392
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027392
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18446744069414584318
[cn4:17832] dstore:hash:store: storing key cn2[OPAL_UINT32] for proc 18446744069414584318
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 9594994688
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994688
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18446744069414584318
[cn4:17832] dstore:hash:store: storing key cn3[OPAL_UINT32] for proc 18446744069414584318
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 13889961984
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961984
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18446744069414584318
[cn4:17832] dstore:hash:store: storing key cn4[OPAL_UINT32] for proc 18446744069414584318
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 31069831169
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 31069831169
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18184929280
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929280
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18446744069414584318
[cn4:17832] dstore:hash:store: storing key cn5[OPAL_UINT32] for proc 18446744069414584318
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18446744070419644417
[cn4:17832] dstore:hash:store: storing key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18446744070419644417
[cn4:17832] dstore:hash:store: updating key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 1005060097
[cn4:17832] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 1005060097
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 1005060097
[cn4:17832] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 1005060097
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 1005060097
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 1005060097
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 1005060097
[cn4:17832] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 1005060097
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 1005060097
[cn4:17832] dstore:has[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 1005060096
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060096
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18446744069414584318
[cn3:21419] dstore:hash:store: storing key cn1[OPAL_UINT32] for proc 18446744069414584318
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 5300027392
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027392
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18446744069414584318
[cn3:21419] dstore:hash:store: storing key cn2[OPAL_UINT32] for proc 18446744069414584318
[cn3:21419] storing data in INTERNAL dstore
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 319
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 319 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2076
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 2076 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 319 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 2076 bytes from [[15336,0],2] for tag 2
[cn2:01045] mca: base: components_open: component v open function successful
[cn2:01045] mca: base: components_open: found loaded component bfo
[cn2:01045] mca: base: components_open: component bfo open function successful
[cn2:01045] mca: base: components_open: found loaded component cm
h:store: storing key orte.hostname[OPAL_STRING] for proc 1005060097
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 1005060097
[cn4:17832] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 1005060097
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 5300027393
[cn4:17832] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 5300027393
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 5300027393
[cn4:17832] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 5300027393
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 5300027393
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 5300027393
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 5300027393
[cn4:17832] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 5300027393
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 5300027393
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027393
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 5300027393
[cn4:17832] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 5300027393
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 9594994689
[cn4:17832] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 9594994689
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 9594994689
[cn4:17832] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 9594994689
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 9594994689
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 9594994689
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 9594994689
[cn4:17832] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 9594994689
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 9594994689
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994689
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 9594994689
[cn4:17832] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 9594994689
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 13889961985
[cn4:17832] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 13889961985
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 13889961985
[cn4:17832] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 13889961985
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 13889961985
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 13889961985
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn4:17832] st[cn3:21419] dstore:hash:store storing data for proc 9594994688
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994688
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18446744069414584318
[cn3:21419] dstore:hash:store: storing key cn3[OPAL_UINT32] for proc 18446744069414584318
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929281
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 18184929281
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 13889961984
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961984
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18446744069414584318
[cn3:21419] dstore:hash:store: storing key cn4[OPAL_UINT32] for proc 18446744069414584318
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929280
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929280
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18446744069414584318
[cn3:21419] dstore:hash:store: storing key cn5[OPAL_UINT32] for proc 18446744069414584318
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18446744070419644417
[cn3:21419] dstore:hash:store: storing key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18446744070419644417
[cn3:21419] dstore:hash:store: updating key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn3:21419] storing data in INTERNAL dstore
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3403
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 3403 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 3403 bytes from [[15336,0],2] for tag 2
oring data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 13889961985
[cn4:17832] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 13889961985
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 13889961985
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961985
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 13889961985
[cn4:17832] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 13889961985
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18184929281
[cn4:17832] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 18184929281
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18184929281
[cn4:17832] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 18184929281
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18184929281
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 18184929281
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 18184929281
[cn4:17832] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 18184929281
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 18184929281
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929281
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18184929281
[cn4:17832] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 18184929281
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 22479896577
[cn4:17832] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 22479896577
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 22479896577
[cn4:17832] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 22479896577
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 22479896577
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 22479896577
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 22479896577
[cn4:17832] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 22479896577
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 22479896577
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 22479896577
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 22479896577
[cn4:17832] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 22479896577
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key opal.local.ldr[OPAL_UINT64] for proc 31069831169
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 26774863873
[cn4:17832] dstore:hash:[cn3:21419] dstore:hash:store storing data for proc 1005060097
[cn3:21419] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 1005060097
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 1005060097
[cn3:21419] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 1005060097
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 1005060097
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 1005060097
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 1005060097
[cn3:21419] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 1005060097
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 1005060097
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060097
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 1005060097
[cn3:21419] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 1005060097
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 5300027393
[cn3:21419] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 5300027393
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 5300027393
[cn3:21419] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 5300027393
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 5300027393
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 5300027393
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 5300027393
[cn3:21419] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 5300027393
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 5300027393
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027393
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 5300027393
[cn3:21419] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 5300027393
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 9594994689
[cn3:21419] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 9594994689
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 9594994689
[cn3:21419] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 9594994689
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2509
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 2509 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 2509 bytes from [[15336,0],2] for tag 2
store: storing key opal.local.rank[OPAL_UINT16] for proc 26774863873
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 26774863873
[cn4:17832] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 26774863873
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 26774863873
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 26774863873
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 26774863873
[cn4:17832] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 26774863873
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 26774863873
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 26774863873
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 26774863873
[cn4:17832] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 26774863873
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 31069831169
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 31069831169
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 31069831169
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 35364798465
[cn4:17832] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 35364798465
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 35364798465
[cn4:17832] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 35364798465
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 35364798465
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 35364798465
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 35364798465
[cn4:17832] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 35364798465
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 35364798465
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 35364798465
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 35364798465
[cn4:17832] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 35364798465
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 39659765761
[cn4:17832] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 9594994689
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 9594994689
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 9594994689
[cn3:21419] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 9594994689
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 9594994689
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994689
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 9594994689
[cn3:21419] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 9594994689
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 13889961985
[cn3:21419] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 13889961985
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 13889961985
[cn3:21419] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 13889961985
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 13889961985
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 13889961985
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 13889961985
[cn3:21419] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 13889961985
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 13889961985
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961985
[cn3:21419] storing data in INTERNAL dstore
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1898
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1898 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3108
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 3108 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1898 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 3108 bytes from [[15336,0],2] for tag 2
 proc 39659765761
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 39659765761
[cn4:17832] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 39659765761
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 39659765761
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 39659765761
[cn4:17832] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 39659765761
[cn4:17832] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 39659765761
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 39659765761
[cn4:17832] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 39659765761
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 39659765761
[cn4:17832] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 39659765761
[cn4:17832] [[15336,1],7] Process bound at launch
[cn4:17832] MCW rank 7 bound to socket 0[core 1[hwt 0]]: [./B/./.][./././.]
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: updating key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn4:17832] storing data in PEER dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 31069831169
[cn3:21419] dstore:hash:store storing data for proc 13889961985
[cn3:21419] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 13889961985
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key opal.local.ldr[OPAL_UINT64] for proc 18184929281
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 18184929281
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 18184929281
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 18184929281
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 22479896577
[cn3:21419] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 22479896577
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 22479896577
[cn3:21419] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 22479896577
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 22479896577
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 22479896577
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 22479896577
[cn3:21419] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 22479896577
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 22479896577
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 22479896577
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 22479896577
[cn3:21419] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 22479896577
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 26774863873
[cn3:21419] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 26774863873
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 464
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 464 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2396
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 2396 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 464 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 2396 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 464
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 464 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1745
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1745 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17832] mca: base: components_register: registering bml components
[cn4:17832] mca: base: components_register: found loaded component r2
[cn4:17832] mca: base: components_register: component r2 register function successful
[cn4:17832] mca: base: components_open: opening bml components
[cn4:17832] mca: base: components_open: found loaded component r2
[cn4:17832] mca: base: components_open: component r2 open function successful
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 26774863873
[cn3:21419] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 26774863873
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 26774863873
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 26774863873
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 26774863873
[cn3:21419] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 26774863873
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 26774863873
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 26774863873
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 26774863873
[cn3:21419] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 26774863873
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 31069831169
[cn3:21419] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 31069831169
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 31069831169
[cn3:21419] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 31069831169
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 31069831169
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 31069831169
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 31069831169
[cn3:21419] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 31069831169
[cn3:21419] fetching data from INTERNAL dstore
[cn1:14209] [[15336,0],0] message received 464 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 1745 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1848
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1848 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] mca: base: components_register: registering bml components
[cn4:17831] mca: base: components_register: found loaded component r2
[cn4:17831] mca: base: components_register: component r2 register function successful
[cn4:17831] mca: base: components_open: opening bml components
[cn4:17831] mca: base: components_open: found loaded component r2
[cn4:17831] mca: base: components_open: component r2 open function successful
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 31069831169
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 31069831169
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 31069831169
[cn3:21419] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 31069831169
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 35364798465
[cn3:21419] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 35364798465
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 35364798465
[cn3:21419] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 35364798465
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 35364798465
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 35364798465
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 35364798465
[cn3:21419] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 35364798465
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 35364798465
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 788
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 788 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1848 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 788 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 788
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 788 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 35364798465
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 35364798465
[cn3:21419] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 35364798465
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 39659765761
[cn3:21419] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 39659765761
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 39659765761
[cn3:21419] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 39659765761
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 39659765761
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 39659765761
[cn3:21419] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 39659765761
[cn3:21419] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 39659765761
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 39659765761
[cn3:21419] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 39659765761
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 39659765761
[cn3:21419] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 39659765761
[cn4:17832] mca: base: components_register: registering btl components
[cn4:17832] mca: base: components_register: found loaded component openib
[cn4:17832] mca: base: components_register: component openib register function successful
[cn4:17832] mca: base: components_register: found loaded component self
[cn4:17832] mca: base: components_register: component self register function successful
[cn4:17832] mca: base: components_open: opening btl components
[cn4:17832] mca: base: components_open: found loaded component openib
[cn4:17832] mca: base: components_open: component openib open function successful
[cn4:17832] mca: base: components_open: found loaded component self
[cn4:17832] mca: base: components_open: component self open function successful
[cn1:14209] [[15336,0],0] message received 788 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],2] for tag 2
[cn4:17831] mca: base: components_register: registering btl components
[cn4:17831] mca: base: components_register: found loaded component openib
[cn4:17831] mca: base: components_register: component openib register function successful
[cn4:17831] mca: base: components_register: found loaded component self
[cn4:17831] mca: base: components_register: component self register function successful
[cn4:17831] mca: base: components_open: opening btl components
[cn4:17831] mca: base: components_open: found loaded component openib
[cn4:17831] mca: base: components_open: component openib open function successful
[cn4:17831] mca: base: components_open: found loaded component self
[cn4:17831] mca: base: components_open: component self open function successful
[cn3:21419] [[15336,1],4] Process bound at launch
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 855
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 855 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 303
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 303 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 855 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 303 bytes from [[15336,0],2] for tag 2
[cn4:17832] mca: base: components_register: registering pml components
[cn4:17832] mca: base: components_register: found loaded component v
[cn4:17832] mca: base: components_register: component v register function successful
[cn4:17832] mca: base: components_register: found loaded component bfo
[cn4:17832] mca: base: components_register: component bfo register function successful
[cn4:17832] mca: base: components_register: found loaded component cm
[cn4:17832] mca: base: components_register: component cm register function successful
[cn4:17832] mca: base: components_register: found loaded component ob1
[cn4:17832] mca: base: components_register: component ob1 register function successful
[cn4:17832] mca: base: components_open: opening pml components
[cn4:17832] mca: base: components_open: found loaded component v
[cn3:21419] MCW rank 4 bound to socket 0[core 0[hwt 0]]: [B/././.][./././.]
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: updating key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 855
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 855 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 233
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 233 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 855 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 233 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 319
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 319 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1242
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1242 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] mca: base: components_register: registering pml components
[cn4:17831] mca: base: components_register: found loaded component v
[cn4:17831] mca: base: components_register: component v register function successful
[cn4:17831] mca: base: components_register: found loaded component bfo
[cn4:17831] mca: base: components_register: component bfo register function successful
[cn4:17831] mca: base: components_register: found loaded component cm
[cn4:17831] mca: base: components_register: component cm register function successful
[cn4:17831] mca: base: components_register: found loaded component ob1
[cn4:17831] mca: base: components_register: component ob1 register function successful
[cn4:17831] mca: base: components_open: opening pml components
[cn4:17831] mca: base: components_open: found loaded component v
[cn3:21419] storing data in PEER dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 18184929281
[cn1:14209] [[15336,0],0] message received 319 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 1242 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 319
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 319 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1672
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1672 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17832] mca: base: components_open: component v open function successful
[cn4:17832] mca: base: components_open: found loaded component bfo
[cn4:17832] mca: base: components_open: component bfo open function successful
[cn4:17832] mca: base: components_open: found loaded component cm
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 1005060096
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060096
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18446744069414584318
[cn3:21420] dstore:hash:store: storing key cn1[OPAL_UINT32] for proc 18446744069414584318
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 5300027392
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027392
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18446744069414584318
[cn3:21420] dstore:hash:store: storing key cn2[OPAL_UINT32] for proc 18446744069414584318
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 9594994688
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994688
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18446744069414584318
[cn3:21420] dstore:hash:store: storing key cn3[OPAL_UINT32] for proc 18446744069414584318
[cn1:14209] [[15336,0],0] message received 319 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 1672 bytes from [[15336,0],2] for tag 2
[cn4:17831] mca: base: components_open: component v open function successful
[cn4:17831] mca: base: components_open: found loaded component bfo
[cn4:17831] mca: base: components_open: component bfo open function successful
[cn4:17831] mca: base: components_open: found loaded component cm
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 22479896577
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 22479896577
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 13889961984
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961984
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18446744069414584318
[cn3:21420] dstore:hash:store: storing key cn4[OPAL_UINT32] for proc 18446744069414584318
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18184929280
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929280
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18446744069414584318
[cn3:21420] dstore:hash:store: storing key cn5[OPAL_UINT32] for proc 18446744069414584318
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18446744070419644417
[cn3:21420] dstore:hash:store: storing key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18446744070419644417
[cn3:21420] dstore:hash:store: updating key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3050
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 3050 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 3050 bytes from [[15336,0],2] for tag 2
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 1005060097
[cn3:21420] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 1005060097
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 1005060097
[cn3:21420] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 1005060097
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 1005060097
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 1005060097
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 1005060097
[cn3:21420] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 1005060097
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 1005060097
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060097
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 1005060097
[cn3:21420] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 1005060097
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 5300027393
[cn3:21420] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 5300027393
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 5300027393
[cn3:21420] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 5300027393
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 5300027393
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 5300027393
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 5300027393
[cn3:21420] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 5300027393
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 5300027393
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027393
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 5300027393
[cn3:21420] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 5300027393
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1648
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1648 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1648 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2968
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 2968 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 9594994689
[cn3:21420] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 9594994689
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 9594994689
[cn3:21420] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 9594994689
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 9594994689
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 9594994689
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 9594994689
[cn3:21420] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 9594994689
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 9594994689
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994689
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 9594994689
[cn3:21420] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 9594994689
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 13889961985
[cn1:14209] [[15336,0],0] message received 2968 bytes from [[15336,0],2] for tag 2
[cn3:21420] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 13889961985
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 13889961985
[cn3:21420] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 13889961985
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 13889961985
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 13889961985
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 13889961985
[cn3:21420] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 13889961985
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 13889961985
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961985
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 13889961985
[cn3:21420] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 13889961985
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key opal.local.ldr[OPAL_UINT64] for proc 22479896577
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18184929281
[cn3:21420] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 18184929281
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18184929281
[cn3:21420] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 18184929281
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18184929281
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 18184929281
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 18184929281
[cn3:21420] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 18184929281
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 18184929281
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929281
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1672
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1672 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1672 bytes from [[15336,0],2] for tag 2
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18184929281
[cn3:21420] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 18184929281
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 22479896577
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 22479896577
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 22479896577
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 26774863873
[cn3:21420] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 26774863873
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 26774863873
[cn3:21420] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 26774863873
[cn3:21420] storing data in INTERNAL dstore
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3036
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 3036 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 3036 bytes from [[15336,0],2] for tag 2
[cn3:21420] dstore:hash:store storing data for proc 26774863873
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 26774863873
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 26774863873
[cn3:21420] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 26774863873
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 26774863873
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 26774863873
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 26774863873
[cn3:21420] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 26774863873
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 31069831169
[cn3:21420] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 31069831169
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 31069831169
[cn3:21420] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 31069831169
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 31069831169
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 31069831169
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 31069831169
[cn3:21420] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 31069831169
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 31069831169
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 31069831169
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 31069831169
[cn3:21420] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 31069831169
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 35364798465
[cn3:21420] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 35364798465
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 35364798465
[cn3:21420] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 35364798465
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2679
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 2679 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 2679 bytes from [[15336,0],2] for tag 2
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 35364798465
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 35364798465
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 35364798465
[cn3:21420] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 35364798465
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 35364798465
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 35364798465
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 35364798465
[cn3:21420] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 35364798465
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 39659765761
[cn3:21420] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 39659765761
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 39659765761
[cn3:21420] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 39659765761
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 39659765761
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 39659765761
[cn3:21420] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 39659765761
[cn3:21420] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 39659765761
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 39659765761
[cn3:21420] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 39659765761
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 39659765761
[cn3:21420] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 39659765761
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],2] for tag 2
[cn3:21420] [[15336,1],5] Process bound at launch
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 343
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 343 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 343 bytes from [[15336,0],2] for tag 2
[cn3:21420] MCW rank 5 bound to socket 0[core 1[hwt 0]]: [./B/./.][./././.]
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: updating key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn3:21420] storing data in PEER dstore
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 193
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 193 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 193 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 464
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 464 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 22479896577
[cn1:14209] [[15336,0],0] message received 464 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 464
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 464 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] mca: base: components_register: registering bml components
[cn3:21419] mca: base: components_register: found loaded component r2
[cn3:21419] mca: base: components_register: component r2 register function successful
[cn3:21419] mca: base: components_open: opening bml components
[cn3:21419] mca: base: components_open: found loaded component r2
[cn3:21419] mca: base: components_open: component r2 open function successful
[cn1:14209] [[15336,0],0] message received 464 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 788
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 788 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] mca: base: components_register: registering bml components
[cn3:21420] mca: base: components_register: found loaded component r2
[cn3:21420] mca: base: components_register: component r2 register function successful
[cn3:21420] mca: base: components_open: opening bml components
[cn3:21420] mca: base: components_open: found loaded component r2
[cn3:21420] mca: base: components_open: component r2 open function successful
[cn1:14209] [[15336,0],0] message received 788 bytes from [[15336,0],2] for tag 2
[cn3:21419] mca: base: components_register: registering btl components
[cn3:21419] mca: base: components_register: found loaded component openib
[cn3:21419] mca: base: components_register: component openib register function successful
[cn3:21419] mca: base: components_register: found loaded component self
[cn3:21419] mca: base: components_register: component self register function successful
[cn3:21419] mca: base: components_open: opening btl components
[cn3:21419] mca: base: components_open: found loaded component openib
[cn3:21419] mca: base: components_open: component openib open function successful
[cn3:21419] mca: base: components_open: found loaded component self
[cn3:21419] mca: base: components_open: component self open function successful
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 788
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 788 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 788 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 855
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 855 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] mca: base: components_register: registering btl components
[cn3:21420] mca: base: components_register: found loaded component openib
[cn3:21420] mca: base: components_register: component openib register function successful
[cn3:21420] mca: base: components_register: found loaded component self
[cn3:21420] mca: base: components_register: component self register function successful
[cn3:21420] mca: base: components_open: opening btl components
[cn3:21420] mca: base: components_open: found loaded component openib
[cn3:21420] mca: base: components_open: component openib open function successful
[cn3:21420] mca: base: components_open: found loaded component self
[cn3:21420] mca: base: components_open: component self open function successful
[cn1:14209] [[15336,0],0] message received 855 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 855
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 855 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] mca: base: components_register: registering pml components
[cn3:21419] mca: base: components_register: found loaded component v
[cn3:21419] mca: base: components_register: component v register function successful
[cn3:21419] mca: base: components_register: found loaded component bfo
[cn3:21419] mca: base: components_register: component bfo register function successful
[cn3:21419] mca: base: components_register: found loaded component cm
[cn3:21419] mca: base: components_register: component cm register function successful
[cn3:21419] mca: base: components_register: found loaded component ob1
[cn3:21419] mca: base: components_register: component ob1 register function successful
[cn3:21419] mca: base: components_open: opening pml components
[cn3:21419] mca: base: components_open: found loaded component v
[cn1:14209] [[15336,0],0] message received 855 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 319
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 319 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] mca: base: components_register: registering pml components
[cn3:21420] mca: base: components_register: found loaded component v
[cn3:21420] mca: base: components_register: component v register function successful
[cn3:21420] mca: base: components_register: found loaded component bfo
[cn3:21420] mca: base: components_register: component bfo register function successful
[cn3:21420] mca: base: components_register: found loaded component cm
[cn3:21420] mca: base: components_register: component cm register function successful
[cn3:21420] mca: base: components_register: found loaded component ob1
[cn3:21420] mca: base: components_register: component ob1 register function successful
[cn3:21420] mca: base: components_open: opening pml components
[cn3:21420] mca: base: components_open: found loaded component v
[cn1:14209] [[15336,0],0] message received 319 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 319
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 319 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] mca: base: components_open: component v open function successful
[cn3:21419] mca: base: components_open: found loaded component bfo
[cn3:21419] mca: base: components_open: component bfo open function successful
[cn3:21419] mca: base: components_open: found loaded component cm
[cn1:14209] [[15336,0],0] message received 319 bytes from [[15336,0],2] for tag 2
[cn3:21420] mca: base: components_open: component v open function successful
[cn3:21420] mca: base: components_open: found loaded component bfo
[cn3:21420] mca: base: components_open: component bfo open function successful
[cn3:21420] mca: base: components_open: found loaded component cm
[cn1:14209] [[15336,0],0] plm:base:launch wiring up iof for job [15336,1]
[cn1:14225] progress: initialized event flag to: 3
[cn1:14225] progress: initialized yield_when_idle to: false
[cn1:14225] progress: initialized num users to: 0
[cn1:14225] progress: initialized poll rate to: 23940000
[cn1:14225] progress: event_users_increment setting count to 1
[cn1:14225] mca: base: components_register: registering ess components
[cn1:14225] mca: base: components_register: found loaded component env
[cn1:14225] mca: base: components_register: component env has no register or open function
[cn1:14225] mca: base: components_open: opening ess components
[cn1:14225] mca: base: components_open: found loaded component env
[cn1:14225] mca: base: components_open: component env open function successful
[cn1:14225] mca:base:select: Auto-selecting ess components
[cn1:14225] mca:base:select:(  ess) Querying component [env]
[cn1:14225] mca:base:select:(  ess) Query of component [env] set priority to 20
[cn1:14225] mca:base:select:(  ess) Selected component [env]
[cn1:14225] ess:env set name to [[15336,1],1]
[cn1:14225] mca: base: components_register: registering oob components
[cn1:14225] mca: base: components_register: found loaded component tcp
[cn1:14225] mca: base: components_register: component tcp register function successful
[cn1:14225] mca: base: components_open: opening oob components
[cn1:14225] mca: base: components_open: found loaded component tcp
[cn1:14225] mca: base: components_open: component tcp open function successful
[cn1:14225] mca:oob:select: checking available component tcp
[cn1:14225] mca:oob:select: Querying component [tcp]
[cn1:14225] oob:tcp: component_available called
[cn1:14225] WORKING INTERFACE 1 KERNEL INDEX 1 FAMILY: V4
[cn1:14225] [[15336,1],1] oob:tcp:init rejecting loopback interface lo
[cn1:14225] WORKING INTERFACE 2 KERNEL INDEX 2 FAMILY: V4
[cn1:14225] [[15336,1],1] oob:tcp:init creating module for V4 address on interface eth0
[cn1:14225] [[15336,1],1] creating OOB-TCP module for interface eth0
[cn1:14225] [[15336,1],1] oob:tcp:init adding 192.168.1.1 to our list of V4 connections
[cn1:14225] WORKING INTERFACE 3 KERNEL INDEX 3 FAMILY: V4
[cn1:14225] [[15336,1],1] oob:tcp:init creating module for V4 address on interface eth1
[cn1:14225] [[15336,1],1] creating OOB-TCP module for interface eth1
[cn1:14225] [[15336,1],1] oob:tcp:init adding 192.168.10.2 to our list of V4 connections
[cn1:14225] [[15336,1],1] TCP STARTUP
[cn1:14225] [[15336,1],1] attempting to bind to IPv4 port 0
[cn1:14225] [[15336,1],1] assigned IPv4 port 57442
[cn1:14225] mca:oob:select: Adding component to end
[cn1:14225] mca:oob:select: Found 1 active transports
[cn1:14225] mca: base: components_register: registering rml components
[cn1:14225] mca: base: components_register: found loaded component oob
[cn1:14225] mca: base: components_register: component oob has no register or open function
[cn1:14225] mca: base: components_open: opening rml components
[cn1:14225] mca: base: components_open: found loaded component oob
[cn1:14225] mca: base: components_open: component oob open function successful
[cn1:14225] orte_rml_base_select: initializing rml component oob
[cn1:14225] mca: base: components_register: registering routed components
[cn1:14225] mca: base: components_register: found loaded component binomial
[cn1:14225] mca: base: components_register: component binomial has no register or open function
[cn1:14225] mca: base: components_register: found loaded component debruijn
[cn1:14225] mca: base: components_register: component debruijn has no register or open function
[cn1:14225] mca: base: components_register: found loaded component direct
[cn1:14225] mca: base: components_register: component direct has no register or open function
[cn1:14225] mca: base: components_register: found loaded component radix
[cn1:14225] mca: base: components_register: component radix register function successful
[cn1:14225] mca: base: components_open: opening routed components
[cn1:14225] mca: base: components_open: found loaded component binomial
[cn1:14225] mca: base: components_open: found loaded component debruijn
[cn1:14225] mca: base: components_open: found loaded component direct
[cn1:14225] mca: base: components_open: found loaded component radix
[cn1:14225] mca:base:select: Auto-selecting routed components
[cn1:14225] mca:base:select:(routed) Querying component [binomial]
[cn1:14225] mca:base:select:(routed) Query of component [binomial] set priority to 70
[cn1:14225] mca:base:select:(routed) Querying component [debruijn]
[cn1:14225] mca:base:select:(routed) Query of component [debruijn] set priority to 0
[cn1:14225] mca:base:select:(routed) Querying component [direct]
[cn1:14225] mca:base:select:(routed) Query of component [direct] set priority to 0
[cn1:14225] mca:base:select:(routed) Querying component [radix]
[cn1:14225] mca:base:select:(routed) Query of component [radix] set priority to 30
[cn1:14225] mca:base:select:(routed) Selected component [binomial]
[cn1:14225] mca: base: close: unloading component debruijn
[cn1:14225] mca: base: close: unloading component direct
[cn1:14225] mca: base: close: unloading component radix
[cn1:14225] orte_routed_base_select: initializing selected component binomial
[cn1:14225] mca: base: components_register: registering dstore components
[cn1:14225] mca: base: components_register: found loaded component hash
[cn1:14225] mca: base: components_register: component hash register function successful
[cn1:14225] mca: base: components_open: opening dstore components
[cn1:14225] mca: base: components_open: found loaded component hash
[cn1:14225] mca:dstore:select: checking available component hash
[cn1:14225] mca:dstore:select: Querying component [hash]
[cn1:14225] mca: base: components_register: registering grpcomm components
[cn1:14225] mca: base: components_register: found loaded component bad
[cn1:14225] mca: base: components_register: component bad has no register or open function
[cn1:14225] mca: base: components_open: opening grpcomm components
[cn1:14225] mca: base: components_open: found loaded component bad
[cn1:14225] mca: base: components_open: component bad open function successful
[cn1:14225] mca:base:select: Auto-selecting grpcomm components
[cn1:14225] mca:base:select:(grpcomm) Querying component [bad]
[cn1:14225] mca:base:select:(grpcomm) Query of component [bad] set priority to 10
[cn1:14225] mca:base:select:(grpcomm) Selected component [bad]
[cn1:14225] [[15336,1],1] grpcomm:base:receive start comm
[cn1:14225] [[15336,1],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 30
[cn1:14225] [[15336,1],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 48
[cn1:14225] [[15336,1],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 9
[cn1:14225] [[15336,1],1] setting up session dir with
	tmpdir: /tmp
	host cn1
[cn1:14225] [[15336,1],1] posting recv
[cn1:14225] [[15336,1],1] posting persistent recv on tag 30 for peer [[WILDCARD],WILDCARD]
[cn1:14225] [[15336,1],1] posting recv
[cn1:14225] [[15336,1],1] posting persistent recv on tag 48 for peer [[WILDCARD],WILDCARD]
[cn1:14225] [[15336,1],1] posting recv
[cn1:14225] [[15336,1],1] posting persistent recv on tag 9 for peer [[WILDCARD],WILDCARD]
[cn1:14225] procdir: /tmp/openmpi-sessions-artpol@cn1_0/15336/1/1
[cn1:14225] jobdir: /tmp/openmpi-sessions-artpol@cn1_0/15336/1
[cn1:14225] top: openmpi-sessions-artpol@cn1_0
[cn1:14225] tmp: /tmp
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key opal.job.session.dir[OPAL_STRING] for proc 5300027393
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key opal.my.session.dir[OPAL_STRING] for proc 5300027393
[cn1:14225] [[15336,1],1] routed_binomial: init routes for proc job [15336,1]
	hnp_uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
	daemon uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn1:14225] [[15336,1],1] [[15336,0],0] with daemon registering
[cn1:14225] [[15336,1],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 20
[cn1:14225] [[15336,1],1] rml_send_buffer to peer [[15336,0],0] at tag 1
[cn1:14225] [[15336,1],1] registering sync waiting for ack
[cn1:14225] [[15336,1],1] waiting on progress thread at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/routed/base/routed_base_fns.c:350
[cn1:14225] [[15336,1],1]: set_addr to uri 1005060096.0;tcp://192.168.1.1,192.168.10.2:37347
[cn1:14225] [[15336,1],1]:set_addr checking if peer [[15336,0],0] is reachable via component tcp
[cn1:14225] [[15336,1],1] oob:tcp: working peer [[15336,0],0] address tcp://192.168.1.1,192.168.10.2:37347
[cn1:14225] [[15336,1],1] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 2 INTERFACE eth0
[cn1:14225] [[15336,1],1] PASSING ADDR 192.168.1.1 TO INTERFACE eth0 AT KERNEL INDEX 2
[cn1:14225] [[15336,1],1]:tcp set addr for peer [[15336,0],0]
[cn1:14225] [[15336,1],1] PEER [[15336,0],0] MAY BE REACHABLE USING MODULE AT KINDEX 3 INTERFACE eth1
[cn1:14225] [[15336,1],1] PASSING ADDR 192.168.10.2 TO INTERFACE eth1 AT KERNEL INDEX 3
[cn1:14225] [[15336,1],1]:tcp set addr for peer [[15336,0],0]
[cn1:14225] [[15336,1],1]: peer [[15336,0],0] is reachable via component tcp
[cn1:14225] [[15336,1],1] posting recv
[cn1:14225] [[15336,1],1] posting non-persistent recv on tag 20 for peer [[WILDCARD],WILDCARD]
[cn1:14225] [[15336,1],1] rml_send_msg to peer [[15336,0],0] at tag 1
[cn1:14225] [[15336,1],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14225] [[15336,1],1]:tcp:processing set_peer cmd for interface eth0
[cn1:14225] [[15336,1],1] SET_PEER ADDING PEER [[15336,0],0]
[cn1:14225] [[15336,1],1] set_peer: peer [[15336,0],0] is listening on interface eth0 net 192.168.1.1 port 37347
[cn1:14225] [[15336,1],1]:tcp:processing set_peer cmd for interface eth1
[cn1:14225] [[15336,1],1] SET_PEER ADDING PEER [[15336,0],0]
[cn1:14225] [[15336,1],1] set_peer: peer [[15336,0],0] is listening on interface eth1 net 192.168.10.2 port 37347
[cn1:14225] [[15336,1],1] oob:base:send to target [[15336,0],0]
[cn1:14225] [[15336,1],1] oob:tcp:send_nb to peer [[15336,0],0]:1
[cn1:14225] [[15336,1],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:1
[cn1:14225] [[15336,1],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:476] queue pending to [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_nb: initiating connection to [[15336,0],0]
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:490] connect to [[15336,0],0]
[cn1:14225] [[15336,1],1] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0
[cn1:14225] [[15336,1],1] oob:tcp:peer creating socket to [[15336,0],0]
[cn1:14225] [[15336,1],1] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] via interface eth0 on socket 11
[cn1:14225] [[15336,1],1] orte_tcp_peer_try_connect: attempting to connect to proc [[15336,0],0] on 192.168.1.1:37347 - 0 retries
[cn1:14209] [[15336,0],0] mca_oob_tcp_listen_thread: new connection: (0, 11) 192.168.1.1:35083
[cn1:14209] [[15336,0],0] connection_handler: working connection (0, 25) 192.168.1.1:35083
[cn1:14209] [[15336,0],0] accept_connection: 192.168.1.1:35083
[cn1:14225] [[15336,1],1] waiting for connect completion to [[15336,0],0] - activating send event
[cn1:14225] [[15336,1],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_handler CONNECTING
[cn1:14225] [[15336,1],1]:tcp:complete_connect called for peer [[15336,0],0] on socket 11
[cn1:14225] [[15336,1],1] tcp_peer_complete_connect: sending ack to [[15336,0],0]
[cn1:14225] [[15336,1],1] SEND CONNECT ACK
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key opal.cred on proc 5300027393
[cn1:14225] dstore_hash:fetch key opal.cred for proc 5300027393 not found
[cn1:14209] [[15336,0],0]:tcp:recv:handler called
[cn1:14209] [[15336,0],0] RECV CONNECT ACK FROM UNKNOWN ON SOCKET 0
[cn1:14209] [[15336,0],0] waiting for connect ack from UNKNOWN
[cn1:14209] [[15336,0],0] connect ack received from UNKNOWN
[cn1:14209] [[15336,0],0] connect-ack recvd from UNKNOWN
[cn1:14209] [[15336,0],0] mca_oob_tcp_recv_connect: connection from new peer
[cn1:14209] [[15336,0],0] connect-ack header from [[15336,1],1] is okay
[cn1:14209] [[15336,0],0] waiting for connect ack from [[15336,1],1]
[cn1:14209] [[15336,0],0] connect ack received from [[15336,1],1]
[cn1:14209] [[15336,0],0] connect-ack version from [[15336,1],1] matches ours
[cn1:14209] [[15336,0],0] connect-ack [[15336,1],1] authenticated
[cn1:14209] [[15336,0],0] tcp:peer_accept called for peer [[15336,1],1] in state UNKNOWN on socket 0
[cn1:14209] [[15336,0],0] SEND CONNECT ACK
[cn1:14209] [[15336,0],0] send blocking of 47 bytes to socket 0
[cn1:14209] [[15336,0],0] blocking send complete to socket 0
[cn1:14209] [[15336,0],0]-[[15336,1],1] tcp_peer_connected on socket 0
[cn1:14209] [[15336,0],0] routed_binomial_update: [[15336,1],1] --> [[15336,1],1]
[cn1:14209] [[15336,0],0]-[[15336,1],1] accepted: 192.168.1.1 - 192.168.1.1 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn1:14225] [[15336,1],1] send blocking of 47 bytes to socket 11
[cn1:14209] [[15336,0],0] tcp:set_module called for peer [[15336,1],1]
[cn1:14225] [[15336,1],1] blocking send complete to socket 11
[cn1:14225] [[15336,1],1] tcp_peer_complete_connect: setting read event on connection to [[15336,0],0]
[cn1:14225] [[15336,1],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14225] [[15336,1],1] RECV CONNECT ACK FROM [[15336,0],0] ON SOCKET 11
[cn1:14225] [[15336,1],1] waiting for connect ack from [[15336,0],0]
[cn1:14225] [[15336,1],1] connect ack received from [[15336,0],0]
[cn1:14225] [[15336,1],1] connect-ack recvd from [[15336,0],0]
[cn1:14225] [[15336,1],1] connect-ack header from [[15336,0],0] is okay
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 75
[cn1:14225] [[15336,1],1] waiting for connect ack from [[15336,0],0]
[cn1:14225] [[15336,1],1] connect ack received from [[15336,0],0]
[cn1:14225] [[15336,1],1] connect-ack version from [[15336,0],0] matches ours
[cn1:14225] [[15336,1],1] connect-ack [[15336,0],0] authenticated
[cn1:14225] [[15336,1],1]-[[15336,0],0] tcp_peer_connected on socket 11
[cn1:14225] [[15336,1],1]-[[15336,0],0] connected: 192.168.1.1 - 192.168.1.1 nodelay 0 sndbuf 262144 rcvbuf 262144 flags 00000802
[cn1:14225] [[15336,1],1]:tcp:recv:handler starting send/recv events
[cn1:14225] [[15336,1],1] tcp:set_module called for peer [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14225] [[15336,1],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 75 BYTES ON SOCKET 11
[cn1:14225] [[15336,1],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,1],0] (ORIGIN [[15336,1],0]) OF 75 BYTES FOR DEST [[15336,0],0] TAG 1
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 75 bytes from [[15336,1],0] for tag 1
[cn1:14209] [[15336,0],0] orted:comm:process_commands() Processing Command: ORTE_DAEMON_SYNC_WANT_NIDMAP
[cn1:14209] [[15336,0],0] orted_recv: received sync+nidmap from local proc [[15336,1],0]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,1],0] at tag 20
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,1],0] at tag 20
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,1],0]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,1],0]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,1],0]:20
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],0]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],0]:20
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,1],0]) --> [[15336,1],0]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,1],0] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],0]
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,1],0]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,1],0]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,1],0] OF 14140 BYTES ON SOCKET 20
[cn1:14209] [[15336,0],0]-[[15336,1],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14224] [[15336,1],0]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14224] [[15336,1],0]:tcp:recv:handler CONNECTED
[cn1:14224] [[15336,1],0]:tcp:recv:handler allocate new recv msg
[cn1:14224] [[15336,1],0]:tcp:recv:handler read hdr
[cn1:14224] [[15336,1],0]:tcp:recv:handler allocate data region of size 14140
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 172 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 172
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 352 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 172 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 172 bytes from [[15336,0],4] for tag 2
[cn5:14174] mca: base: components_register: registering mtl components
[cn5:14174] mca: base: components_register: found loaded component psm
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 279 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 352
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 352 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 352 bytes from [[15336,0],4] for tag 2
[cn5:14174] mca: base: components_register: component psm register function successful
[cn5:14174] mca: base: components_open: opening mtl components
[cn5:14174] mca: base: components_open: found loaded component psm
[cn5:14174] mca: base: close: component psm closed
[cn5:14174] mca: base: close: unloading component psm
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 279
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 279 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 279 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 172
[cn5:14174] mca: base: close: component cm closed
[cn5:14174] mca: base: close: unloading component cm
[cn5:14174] mca: base: components_open: found loaded component ob1
[cn5:14174] mca: base: components_open: component ob1 open function successful
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 172 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 352 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 279 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 172 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 172 bytes from [[15336,0],4] for tag 2
[cn5:14175] mca: base: components_register: registering mtl components
[cn5:14175] mca: base: components_register: found loaded component psm
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 352
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 352 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 352 bytes from [[15336,0],4] for tag 2
[cn5:14175] mca: base: components_register: component psm register function successful
[cn5:14175] mca: base: components_open: opening mtl components
[cn5:14175] mca: base: components_open: found loaded component psm
[cn5:14175] mca: base: close: component psm closed
[cn5:14175] mca: base: close: unloading component psm
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 279
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 279 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 279 bytes from [[15336,0],4] for tag 2
[cn5:14175] mca: base: close: component cm closed
[cn5:14175] mca: base: close: unloading component cm
[cn5:14175] mca: base: components_open: found loaded component ob1
[cn5:14175] mca: base: components_open: component ob1 open function successful
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 743 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 743 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 743
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 743 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 743 bytes from [[15336,0],1] for tag 2
[cn2:01046] mca: base: components_register: registering mtl components
[cn2:01046] mca: base: components_register: found loaded component psm
[cn2:01046] mca: base: components_register: component psm register function successful
[cn2:01046] mca: base: components_open: opening mtl components
[cn2:01046] mca: base: components_open: found loaded component psm
[cn2:01046] mca: base: close: component psm closed
[cn2:01046] mca: base: close: unloading component psm
[cn2:01046] mca: base: close: component cm closed
[cn2:01046] mca: base: close: unloading component cm
[cn2:01046] mca: base: components_open: found loaded component ob1
[cn2:01046] mca: base: components_open: component ob1 open function successful
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 743
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 743 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 743 bytes from [[15336,0],1] for tag 2
[cn2:01045] mca: base: components_register: registering mtl components
[cn2:01045] mca: base: components_register: found loaded component psm
[cn2:01045] mca: base: components_register: component psm register function successful
[cn2:01045] mca: base: components_open: opening mtl components
[cn2:01045] mca: base: components_open: found loaded component psm
[cn2:01045] mca: base: close: component psm closed
[cn2:01045] mca: base: close: unloading component psm
[cn2:01045] mca: base: close: component cm closed
[cn2:01045] mca: base: close: unloading component cm
[cn2:01045] mca: base: components_open: found loaded component ob1
[cn2:01045] mca: base: components_open: component ob1 open function successful
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 172 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 352 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 172
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 172 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] mca: base: components_register: registering mtl components
[cn1:14209] [[15336,0],0] message received 172 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 279 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 101 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 352 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 279 BYTES ON SOCKET 9
[cn3:21420] mca: base: components_register: registering mtl components
[cn3:21420] mca: base: components_register: found loaded component psm
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 352
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 352 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 352 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 279
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 279 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] mca: base: components_register: component psm register function successful
[cn3:21420] mca: base: components_open: opening mtl components
[cn3:21420] mca: base: components_open: found loaded component psm
[cn3:21420] mca: base: close: component psm closed
[cn3:21420] mca: base: close: unloading component psm
[cn1:14209] [[15336,0],0] message received 279 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 101
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 101 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] mca: base: close: component cm closed
[cn3:21420] mca: base: close: unloading component cm
[cn3:21420] mca: base: components_open: found loaded component ob1
[cn3:21420] mca: base: components_open: component ob1 open function successful
[cn1:14209] [[15336,0],0] message received 101 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 352
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 352 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] mca: base: components_register: found loaded component psm
[cn1:14209] [[15336,0],0] message received 352 bytes from [[15336,0],2] for tag 2
[cn3:21419] mca: base: components_register: component psm register function successful
[cn3:21419] mca: base: components_open: opening mtl components
[cn3:21419] mca: base: components_open: found loaded component psm
[cn3:21419] mca: base: close: component psm closed
[cn3:21419] mca: base: close: unloading component psm
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 279
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 279 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 279 bytes from [[15336,0],2] for tag 2
[cn3:21419] mca: base: close: component cm closed
[cn3:21419] mca: base: close: unloading component cm
[cn3:21419] mca: base: components_open: found loaded component ob1
[cn3:21419] mca: base: components_open: component ob1 open function successful
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 172
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 172 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 172 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 172 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 352 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 352
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 352 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] mca: base: components_register: registering mtl components
[cn4:17831] mca: base: components_register: found loaded component psm
[cn1:14209] [[15336,0],0] message received 352 bytes from [[15336,0],3] for tag 2
[cn4:17831] mca: base: components_register: component psm register function successful
[cn4:17831] mca: base: components_open: opening mtl components
[cn4:17831] mca: base: components_open: found loaded component psm
[cn4:17831] mca: base: close: component psm closed
[cn4:17831] mca: base: close: unloading component psm
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 279
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 279 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 279 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 172
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 172 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] mca: base: close: component cm closed
[cn4:17831] mca: base: close: unloading component cm
[cn4:17831] mca: base: components_open: found loaded component ob1
[cn4:17831] mca: base: components_open: component ob1 open function successful
[cn1:14209] [[15336,0],0] message received 172 bytes from [[15336,0],3] for tag 2
[cn4:17832] mca: base: components_register: registering mtl components
[cn4:17832] mca: base: components_register: found loaded component psm
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 247
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 247 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 279 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 172 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 247 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 305 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 247 bytes from [[15336,0],3] for tag 2
[cn4:17832] mca: base: components_register: component psm register function successful
[cn4:17832] mca: base: components_open: opening mtl components
[cn4:17832] mca: base: components_open: found loaded component psm
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 305
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 305 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 305 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 109
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 109 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17832] mca: base: close: component psm closed
[cn4:17832] mca: base: close: unloading component psm
[cn4:17832] mca: base: close: component cm closed
[cn4:17832] mca: base: close: unloading component cm
[cn4:17832] mca: base: components_open: found loaded component ob1
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 109 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 109 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17832] mca: base: components_open: component ob1 open function successful
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 86
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 86 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 86 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 86 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 249
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 249 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14174] select: component v not in the include list
[cn1:14209] [[15336,0],0] message received 249 bytes from [[15336,0],4] for tag 2
[cn5:14175] select: component v not in the include list
[cn5:14175] select: component bfo not in the include list
[cn5:14175] select: initializing pml component ob1
[cn5:14175] select: initializing btl component openib
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 139
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 249 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 139 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 139 bytes from [[15336,0],4] for tag 2
[cn5:14174] select: component bfo not in the include list
[cn5:14174] select: initializing pml component ob1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 84
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 84 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 84 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14174] select: initializing btl component openib
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 139 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 84 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 86
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 86 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 86 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 86 bytes from [[15336,0],1] for tag 2
[cn2:01046] select: component v not in the include list
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 249
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 249 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 249 bytes from [[15336,0],1] for tag 2
[cn2:01045] select: component v not in the include list
[cn2:01045] select: component bfo not in the include list
[cn2:01045] select: initializing pml component ob1
[cn2:01045] select: initializing btl component openib
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 193
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 193 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 193 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01046] select: component bfo not in the include list
[cn2:01046] select: initializing pml component ob1
[cn2:01046] select: initializing btl component openib
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 249 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 193 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 86
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 86 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 195 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 84 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 86 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 86 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 139 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 195
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 195 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21419] select: component v not in the include list
[cn1:14209] [[15336,0],0] message received 195 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21420] select: component v not in the include list
[cn3:21420] select: component bfo not in the include list
[cn3:21420] select: initializing pml component ob1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 84
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 84 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 84 bytes from [[15336,0],2] for tag 2
[cn3:21420] select: initializing btl component openib
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 139
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 139 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 139 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 84 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 84
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 84 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] select: component bfo not in the include list
[cn3:21419] select: initializing pml component ob1
[cn1:14209] [[15336,0],0] message received 84 bytes from [[15336,0],2] for tag 2
[cn3:21419] select: initializing btl component openib
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,1],1] (ORIGIN [[15336,1],1]) OF 75 BYTES FOR DEST [[15336,0],0] TAG 1
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 75 bytes from [[15336,1],1] for tag 1
[cn1:14209] [[15336,0],0] orted:comm:process_commands() Processing Command: ORTE_DAEMON_SYNC_WANT_NIDMAP
[cn1:14209] [[15336,0],0] orted_recv: received sync+nidmap from local proc [[15336,1],1]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 86 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 86 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 193 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 88 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 81 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,1],1] at tag 20
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,1],1] at tag 20
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,1],1]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,1],1]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,1],1]:20
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],1]:20
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,1],1]) --> [[15336,1],1]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,1],1] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 86
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,1],1]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,1],1]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,1],1] OF 14140 BYTES ON SOCKET 0
[cn1:14209] [[15336,0],0]-[[15336,1],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] plm:base:launch [15336,1] registered
[cn1:14209] [[15336,0],0] plm:base:launch job [15336,1] is not a dynamic spawn
[cn1:14209] [[15336,0],0]: Setting up debugger process table for applications
  MPIR_being_debugged = 0
  MPIR_debug_state = 1
  MPIR_partial_attach_ok = 1
  MPIR_i_am_starter = 0
  MPIR_forward_output = 0
  MPIR_proctable_size = 10
  MPIR_proctable:
    (i, host, exe, pid) = (0, cn1, /home/research/artpol/ompi_dev/app/./hellompi, 14224)
    (i, host, exe, pid) = (1, cn1, /home/research/artpol/ompi_dev/app/./hellompi, 14225)
    (i, host, exe, pid) = (2, cn2, /home/research/artpol/ompi_dev/app/./hellompi, 1045)
    (i, host, exe, pid) = (3, cn2, /home/research/artpol/ompi_dev/app/./hellompi, 1046)
    (i, host, exe, pid) = (4, cn3, /home/research/artpol/ompi_dev/app/./hellompi, 21419)
    (i, host, exe, pid) = (5, cn3, /home/research/artpol/ompi_dev/app/./hellompi, 21420)
    (i, host, exe, pid) = (6, cn4, /home/research/artpol/ompi_dev/app/./hellompi, 17831)
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 84 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
    (i, host, exe, pid) = (7, cn4, /home/research/artpol/ompi_dev/app/./hellompi, 17832)
    (i, host, exe, pid) = (8, cn5, /home/research/artpol/ompi_dev/app/./hellompi, 14174)
    (i, host, exe, pid) = (9, cn5, /home/research/artpol/ompi_dev/app/./hellompi, 14175)
MPIR_executable_path: NULL
MPIR_server_arguments: NULL
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 86 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 86 bytes from [[15336,0],3] for tag 2
[cn1:14225] [[15336,1],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14225] [[15336,1],1]:tcp:recv:handler CONNECTED
[cn1:14225] [[15336,1],1]:tcp:recv:handler allocate new recv msg
[cn1:14225] [[15336,1],1]:tcp:recv:handler read hdr
[cn1:14225] [[15336,1],1]:tcp:recv:handler allocate data region of size 14140
[cn4:17832] select: component v not in the include list
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 86
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 86 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 86 bytes from [[15336,0],3] for tag 2
[cn4:17831] select: component v not in the include list
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 193
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 193 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 193 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 88
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 88 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] select: component bfo not in the include list
[cn4:17831] select: initializing pml component ob1
[cn4:17831] select: initializing btl component openib
[cn1:14209] [[15336,0],0] message received 88 bytes from [[15336,0],3] for tag 2
[cn4:17832] select: component bfo not in the include list
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 81
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 81 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 81 bytes from [[15336,0],3] for tag 2
[cn4:17832] select: initializing pml component ob1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 84
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 84 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 84 bytes from [[15336,0],3] for tag 2
[cn4:17832] select: initializing btl component openib
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 194
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 194 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 194 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:686:init_one_port] looking for mlx4_0:1 GID index 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 219
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 219 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 219 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:717:init_one_port] my IB subnet_id for HCA mlx4_0 port 1 is fe80000000000000
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 79
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 79 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 79 bytes from [[15336,0],4] for tag 2
[cn5:14174] [rank=8] openib: using device mlx4_0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1428
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1428 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1428 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1294:setup_qps] pp: rd_num is 256 rd_low is 192 rd_win 128 rd_rsv 4
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:2060:rdmacm_component_query] rdmacm_component_query
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_ip.c:132:mca_btl_openib_rdma_get_ipv4addr] Looking for mlx4_0:1 in IP address list
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:1948:ipaddrcheck] *** Could not find IP address for mlx4_0:1 -- is there an IP address configured for this device?
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 194 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 219 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 79 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1428 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:543:udcm_module_init] created cpc module 0x11301e0 for btl 0x113e180
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],4] for tag 2
[cn5:14174] openib BTL: rdmacm IP address not found on port
[cn5:14174] openib BTL: rdmacm CPC unavailable for use on mlx4_0:1; skipped
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 216 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 216
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 216 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 216 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:793:udcm_module_create_listen_qp] creating listen QP on port 1
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 194
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 194 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 194 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 194 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:686:init_one_port] looking for mlx4_0:1 GID index 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 219
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 219 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 79 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 219 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 219 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:717:init_one_port] my IB subnet_id for HCA mlx4_0 port 1 is fe80000000000000
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 79
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 79 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 79 bytes from [[15336,0],1] for tag 2
[cn2:01045] [rank=2] openib: using device mlx4_0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 828
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 828 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 828 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1294:setup_qps] pp: rd_num is 256 rd_low is 192 rd_win 128 rd_rsv 4
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 828 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 822 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 822
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 822 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] openib BTL: rdmacm IP address not found on port
[cn2:01045] openib BTL: rdmacm CPC unavailable for use on mlx4_0:1; skipped
[cn1:14209] [[15336,0],0] message received 822 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:2060:rdmacm_component_query] rdmacm_component_query
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_ip.c:132:mca_btl_openib_rdma_get_ipv4addr] Looking for mlx4_0:1 in IP address list
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:1948:ipaddrcheck] *** Could not find IP address for mlx4_0:1 -- is there an IP address configured for this device?
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:543:udcm_module_init] created cpc module 0x1a544a0 for btl 0x1a60580
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 216
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 216 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 216 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 216 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:793:udcm_module_create_listen_qp] creating listen QP on port 1
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 194
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 194 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 194 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 194 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:686:init_one_port] looking for mlx4_0:1 GID index 0
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 932
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 932 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 932 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14174] storing data in PEER dstore
[cn5:14174] dstore:hash:store storing data for proc 35364798465
[cn5:14174] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 35364798465
[cn5:14174] select: init of component openib returned success
[cn5:14174] select: initializing btl component self
[cn5:14174] select: init of component self returned success
[cn5:14174] select: init returned priority 20
[cn5:14174] selected ob1 best priority 20
[cn5:14174] select: component ob1 selected
[cn5:14174] mca: base: close: component v closed
[cn5:14174] mca: base: close: unloading component v
[cn5:14174] mca: base: close: component bfo closed
[cn5:14174] mca: base: close: unloading component bfo
[cn5:14174] [[15336,1],8] running modex
[cn5:14174] [[15336,1],8] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:325
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 215
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 215 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 215 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 932 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14175] [rank=9] openib: using device mlx4_0
[cn5:14175] openib BTL: rdmacm IP address not found on port
[cn5:14175] openib BTL: rdmacm CPC unavailable for use on mlx4_0:1; skipped
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],8]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 596
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 139
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 596 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 596 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 215 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1807
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 596 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1807 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:854:udcm_module_create_listen_qp] listening for connections on lid 8, qpn 27543
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:598:udcm_module_init] my modex = LID: 8, Port: 1, QPN: 27543
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:472:udcm_component_query] available for use on mlx4_0:1
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1807 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 1807 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2619
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 2619 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:717:init_one_port] my IB subnet_id for HCA mlx4_0 port 1 is fe80000000000000
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1294:setup_qps] pp: rd_num is 256 rd_low is 192 rd_win 128 rd_rsv 4
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:2060:rdmacm_component_query] rdmacm_component_query
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_ip.c:132:mca_btl_openib_rdma_get_ipv4addr] Looking for mlx4_0:1 in IP address list
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:1948:ipaddrcheck] *** Could not find IP address for mlx4_0:1 -- is there an IP address configured for this device?
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:543:udcm_module_init] created cpc module 0xb691a0 for btl 0xb77140
[cn1:14209] [[15336,0],0] message received 2619 bytes from [[15336,0],4] for tag 2
[cn5:14174] [[15336,1],8] grpcomm:base:modex: performing modex
[cn5:14174] fetching data from PEER dstore
[cn5:14174] dstore:hash:fetch: searching for key NULL on proc 35364798465
[cn5:14174] dstore:hash:fetch: adding data for key MPI_THREAD_LEVEL on proc 35364798465
[cn5:14174] dstore:hash:fetch: adding data for key btl.openib.1.9 on proc 35364798465
[cn5:14174] [[15336,1],8] grpcomm:base:pack_modex: reporting 2 entries
[cn5:14174] [[15336,1],8] grpcomm:base:pack_modex: packing entry for MPI_THREAD_LEVEL
[cn5:14174] [[15336,1],8] grpcomm:base:pack_modex: packing entry for btl.openib.1.9
[cn5:14174] [[15336,1],8] grpcomm:base:full:modex: executing allgather
[cn5:14174] [[15336,1],8] grpcomm:bad entering allgather
[cn5:14174] [[15336,1],8] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:315
[cn5:14174] [[15336,1],8] grpcomm:base:modex: modex posted
[cn5:14174] [[15336,1],8] grpcomm:bad sending collective 0 to our daemon
[cn5:14174] [[15336,1],8] rml_send_buffer to peer [[15336,0],4] at tag 30
[cn5:14174] [[15336,1],8] grpcomm:bad allgather underway
[cn5:14174] [[15336,1],8] rml_send_msg to peer [[15336,0],4] at tag 30
[cn5:14174] [[15336,1],8] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14174] [[15336,1],8] oob:base:send to target [[15336,0],4]
[cn5:14174] [[15336,1],8] oob:base:send known transport for peer [[15336,0],4]
[cn5:14174] [[15336,1],8] oob:tcp:send_nb to peer [[15336,0],4]:30
[cn5:14174] [[15336,1],8] tcp:send_nb to peer [[15336,0],4]
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:30
[cn5:14174] [[15336,1],8] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_nb: already connected to [[15336,0],4] - queueing for send
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_handler called to send to peer [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_handler SENDING TO [[15336,0],4]
[cn5:14174] [[15336,1],8] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 139 BYTES ON SOCKET 11
[cn5:14174] [[15336,1],8]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2619 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 216
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 216 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 216 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 216 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:793:udcm_module_create_listen_qp] creating listen QP on port 1
[cn1:14224] [[15336,1],0]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14224] [[15336,1],0]:tcp:recv:handler CONNECTED
[cn1:14224] [[15336,1],0] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 14140 BYTES FOR DEST [[15336,1],0] TAG 20
[cn1:14224] [[15336,1],0] DELIVERING TO RML
[cn1:14224] [[15336,1],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14224] [[15336,1],0] message received 14140 bytes from [[15336,0],0] for tag 20
[cn1:14224] [[15336,1],0] registering sync ack recvd
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 194
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 194 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 194 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 194 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 932
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 932 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:686:init_one_port] looking for mlx4_0:1 GID index 0
[cn1:14209] [[15336,0],0] message received 932 bytes from [[15336,0],4] for tag 2
[cn5:14175] storing data in PEER dstore
[cn5:14175] dstore:hash:store storing data for proc 39659765761
[cn5:14175] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 39659765761
[cn5:14175] select: init of component openib returned success
[cn5:14175] select: initializing btl component self
[cn5:14175] select: init of component self returned success
[cn5:14175] select: init returned priority 20
[cn5:14175] selected ob1 best priority 20
[cn5:14175] select: component ob1 selected
[cn5:14175] mca: base: close: component v closed
[cn5:14175] mca: base: close: unloading component v
[cn5:14175] mca: base: close: component bfo closed
[cn5:14175] mca: base: close: unloading component bfo
[cn5:14175] [[15336,1],9] running modex
[cn5:14175] [[15336,1],9] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:325
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 596
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 596 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 596 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:854:udcm_module_create_listen_qp] listening for connections on lid 8, qpn 27544
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:598:udcm_module_init] my modex = LID: 8, Port: 1, QPN: 27544
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:472:udcm_component_query] available for use on mlx4_0:1
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 932 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2619
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 219
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 219 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 219 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 596 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:717:init_one_port] my IB subnet_id for HCA mlx4_0 port 1 is fe80000000000000
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],9]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 79
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 79 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 139
[cn1:14209] [[15336,0],0] message received 79 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 219 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21419] [rank=4] openib: using device mlx4_0
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 79 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 2619 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1573
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 2619 bytes from [[15336,0],4] for tag 2
[cn5:14175] [[15336,1],9] grpcomm:base:modex: performing modex
[cn5:14175] fetching data from PEER dstore
[cn5:14175] dstore:hash:fetch: searching for key NULL on proc 39659765761
[cn5:14175] dstore:hash:fetch: adding data for key MPI_THREAD_LEVEL on proc 39659765761
[cn5:14175] dstore:hash:fetch: adding data for key btl.openib.1.9 on proc 39659765761
[cn5:14175] [[15336,1],9] grpcomm:base:pack_modex: reporting 2 entries
[cn5:14175] [[15336,1],9] grpcomm:base:pack_modex: packing entry for MPI_THREAD_LEVEL
[cn5:14175] [[15336,1],9] grpcomm:base:pack_modex: packing entry for btl.openib.1.9
[cn5:14175] [[15336,1],9] grpcomm:base:full:modex: executing allgather
[cn5:14175] [[15336,1],9] grpcomm:bad entering allgather
[cn5:14175] [[15336,1],9] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:315
[cn5:14175] [[15336,1],9] grpcomm:base:modex: modex posted
[cn5:14175] [[15336,1],9] grpcomm:bad sending collective 0 to our daemon
[cn5:14175] [[15336,1],9] rml_send_buffer to peer [[15336,0],4] at tag 30
[cn5:14175] [[15336,1],9] grpcomm:bad allgather underway
[cn5:14175] [[15336,1],9] rml_send_msg to peer [[15336,0],4] at tag 30
[cn5:14175] [[15336,1],9] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14175] [[15336,1],9] oob:base:send to target [[15336,0],4]
[cn5:14175] [[15336,1],9] oob:base:send known transport for peer [[15336,0],4]
[cn5:14175] [[15336,1],9] oob:tcp:send_nb to peer [[15336,0],4]:30
[cn5:14175] [[15336,1],9] tcp:send_nb to peer [[15336,0],4]
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:30
[cn5:14175] [[15336,1],9] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_nb: already connected to [[15336,0],4] - queueing for send
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_handler called to send to peer [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_handler SENDING TO [[15336,0],4]
[cn5:14175] [[15336,1],9] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 139 BYTES ON SOCKET 11
[cn5:14175] [[15336,1],9]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1573 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1573 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 1573 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1294:setup_qps] pp: rd_num is 256 rd_low is 192 rd_win 128 rd_rsv 4
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:2060:rdmacm_component_query] rdmacm_component_query
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_ip.c:132:mca_btl_openib_rdma_get_ipv4addr] Looking for mlx4_0:1 in IP address list
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:1948:ipaddrcheck] *** Could not find IP address for mlx4_0:1 -- is there an IP address configured for this device?
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:543:udcm_module_init] [cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2619 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 75
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 75 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 75 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] openib BTL: rdmacm IP address not found on port
[cn3:21419] openib BTL: rdmacm CPC unavailable for use on mlx4_0:1; skipped
[cn1:14209] [[15336,0],0] message received 75 bytes from [[15336,0],2] for tag 2
created cpc module 0xe7f420 for btl 0xe8d120
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 194
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 194 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 194 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:686:init_one_port] looking for mlx4_0:1 GID index 0
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 194 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 219
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 219 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 219 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:717:init_one_port] my IB subnet_id for HCA mlx4_0 port 1 is fe80000000000000
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 79
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 79 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 79 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 216
[cn2:01046] [rank=3] openib: using device mlx4_0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 828
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 828 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 828 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1294:setup_qps] pp: rd_num is 256 rd_low is 192 rd_win 128 rd_rsv 4
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 820
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 820 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 216 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] openib BTL: rdmacm IP address not found on port
[cn2:01046] openib BTL: rdmacm CPC unavailable for use on mlx4_0:1; skipped
[cn1:14209] [[15336,0],0] message received 820 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 216 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:2060:rdmacm_component_query] rdmacm_component_query
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_ip.c:132:mca_btl_openib_rdma_get_ipv4addr] Looking for mlx4_0:1 in IP address list
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:1948:ipaddrcheck] *** Could not find IP address for mlx4_0:1 -- is there an IP address configured for this device?
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:543:udcm_module_init] created cpc module 0xa2d460 for btl 0xa3b160
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:793:udcm_module_create_listen_qp] creating listen QP on port 1
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060096
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060096
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18446744069414584318
[cn1:14224] dstore:hash:store: storing key cn1[OPAL_UINT32] for proc 18446744069414584318
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060097
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 1005060097
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 5300027392
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027392
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18446744069414584318
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 417
[cn1:14224] dstore:hash:store: storing key cn2[OPAL_UINT32] for proc 18446744069414584318
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 9594994688
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994688
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18446744069414584318
[cn1:14224] dstore:hash:store: storing key cn3[OPAL_UINT32] for proc 18446744069414584318
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 13889961984
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961984
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18446744069414584318
[cn1:14224] dstore:hash:store: storing key cn4[OPAL_UINT32] for proc 18446744069414584318
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18184929280
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929280
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18446744069414584318
[cn1:14224] dstore:hash:store: storing key cn5[OPAL_UINT32] for proc 18446744069414584318
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18446744070419644417
[cn1:14224] dstore:hash:store: storing key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18446744070419644417
[cn1:14224] dstore:hash:store: updating key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key opal.local.ldr[OPAL_UINT64] for proc 1005060097
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 1005060097
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 1005060097
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 1005060097
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 5300027393
[cn1:14224] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 5300027393
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 5300027393
[cn1:14224] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 5300027393
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 5300027393
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 5300027393
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 417 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 417 bytes from [[15336,0],1] for tag 2
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 5300027393
[cn1:14224] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 5300027393
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 5300027393
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:854:udcm_module_create_listen_qp] listening for connections on lid 4, qpn 37962
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:598:udcm_module_init] my modex = LID: 4, Port: 1, QPN: 37962
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027393
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 5300027393
[cn1:14224] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 5300027393
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 9594994689
[cn1:14224] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 9594994689
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 9594994689
[cn1:14224] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 9594994689
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 9594994689
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 9594994689
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 9594994689
[cn1:14224] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 9594994689
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 9594994689
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994689
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 9594994689
[cn1:14224] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 9594994689
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 13889961985
[cn1:14224] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 13889961985
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 13889961985
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 133
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 133 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 133 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 209
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 209 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14224] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 13889961985
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 13889961985
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 13889961985
[cn2:01045] storing data in PEER dstore
[cn2:01045] dstore:hash:store storing data for proc 9594994689
[cn1:14209] [[15336,0],0] message received 209 bytes from [[15336,0],1] for tag 2
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 13889961985
[cn1:14224] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 13889961985
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 13889961985
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961985
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 13889961985
[cn1:14224] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 13889961985
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18184929281
[cn1:14224] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 18184929281
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18184929281
[cn1:14224] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 18184929281
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18184929281
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 18184929281
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 18184929281
[cn1:14224] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 18184929281
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 18184929281
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929281
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18184929281
[cn1:14224] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 18184929281
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 22479896577
[cn1:14224] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 22479896577
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 22479896577
[cn1:14224] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 22479896577
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 22479896577
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 22479896577
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 22479896577
[cn1:14224] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 22479896577
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:472:udcm_component_query] available for use on mlx4_0:1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 637
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 637 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 637 bytes from [[15336,0],1] for tag 2
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 22479896577
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 22479896577
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 22479896577
[cn1:14224] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 22479896577
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 26774863873
[cn1:14224] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 26774863873
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 26774863873
[cn1:14224] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 26774863873
[cn2:01045] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 9594994689
[cn2:01045] select: init of component openib returned success
[cn2:01045] select: initializing btl component self
[cn2:01045] select: init of component self returned success
[cn2:01045] select: init returned priority 20
[cn2:01045] selected ob1 best priority 20
[cn2:01045] select: component ob1 selected
[cn2:01045] mca: base: close: component v closed
[cn2:01045] mca: base: close: unloading component v
[cn2:01045] mca: base: close: component bfo closed
[cn2:01045] mca: base: close: unloading component bfo
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 812
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 812 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 812 bytes from [[15336,0],1] for tag 2
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 26774863873
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 26774863873
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 26774863873
[cn1:14224] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 26774863873
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 26774863873
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 26774863873
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 26774863873
[cn1:14224] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 26774863873
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 31069831169
[cn1:14224] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 31069831169
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 31069831169
[cn1:14224] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 31069831169
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 31069831169
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 31069831169
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 31069831169
[cn1:14224] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 31069831169
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 31069831169
[cn2:01045] [[15336,1],2] running modex
[cn2:01045] [[15336,1],2] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:325
[cn2:01045] [[15336,1],2] grpcomm:base:modex: performing modex
[cn2:01045] fetching data from PEER dstore
[cn2:01045] dstore:hash:fetch: searching for key NULL on proc 9594994689
[cn2:01045] dstore:hash:fetch: adding data for key MPI_THREAD_LEVEL on proc 9594994689
[cn2:01045] dstore:hash:fetch: adding data for key btl.openib.1.9 on proc 9594994689
[cn2:01045] [[15336,1],2] grpcomm:base:pack_modex: reporting 2 entries
[cn2:01045] [[15336,1],2] grpcomm:base:pack_modex: packing entry for MPI_THREAD_LEVEL
[cn2:01045] [[15336,1],2] grpcomm:base:pack_modex: packing entry for btl.openib.1.9
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 31069831169
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 31069831169
[cn1:14224] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 31069831169
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 35364798465
[cn1:14224] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 35364798465
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 35364798465
[cn1:14224] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 35364798465
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 35364798465
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 35364798465
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 35364798465
[cn1:14224] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 35364798465
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 35364798465
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 35364798465
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 35364798465
[cn1:14224] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 35364798465
[cn1:14224] storing data in INTERNAL dstore
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 764
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 764 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14224] dstore:hash:store storing data for proc 39659765761
[cn1:14224] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 39659765761
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 39659765761
[cn1:14224] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 39659765761
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 39659765761
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn1:14209] [[15336,0],0] message received 764 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1290
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1290 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 39659765761
[cn1:14224] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 39659765761
[cn1:14224] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 39659765761
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 39659765761
[cn1:14224] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 39659765761
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 39659765761
[cn1:14224] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 39659765761
[cn1:14224] [[15336,1],0] Process bound at launch
[cn2:01045] [[15336,1],2] grpcomm:base:full:modex: executing allgather
[cn2:01045] [[15336,1],2] grpcomm:bad entering allgather
[cn2:01045] [[15336,1],2] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:315
[cn2:01045] [[15336,1],2] grpcomm:base:modex: modex posted
[cn2:01045] [[15336,1],2] grpcomm:bad sending collective 0 to our daemon
[cn2:01045] [[15336,1],2] rml_send_buffer to peer [[15336,0],1] at tag 30
[cn2:01045] [[15336,1],2] grpcomm:bad allgather underway
[cn2:01045] [[15336,1],2] rml_send_msg to peer [[15336,0],1] at tag 30
[cn2:01045] [[15336,1],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 1290 bytes from [[15336,0],1] for tag 2
[cn2:01045] [[15336,1],2] oob:base:send to target [[15336,0],1]
[cn2:01045] [[15336,1],2] oob:base:send known transport for peer [[15336,0],1]
[cn2:01045] [[15336,1],2] oob:tcp:send_nb to peer [[15336,0],1]:30
[cn2:01045] [[15336,1],2] tcp:send_nb to peer [[15336,0],1]
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:30
[cn2:01045] [[15336,1],2] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_handler called to send to peer [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_handler SENDING TO [[15336,0],1]
[cn2:01045] [[15336,1],2] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 139 BYTES ON SOCKET 11
[cn2:01045] [[15336,1],2]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 216
[cn1:14224] MCW rank 0 bound to socket 0[core 0[hwt 0]]: [B/././.][./././.]
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: updating key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn1:14224] storing data in PEER dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 1005060097
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 216 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 216 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:793:udcm_module_create_listen_qp] creating listen QP on port 1
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 219 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 79 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 216 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 828 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 820 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 417 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 932
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 932 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 932 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01046] storing data in PEER dstore
[cn2:01046] dstore:hash:store storing data for proc 13889961985
[cn2:01046] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 13889961985
[cn2:01046] select: init of component openib returned success
[cn2:01046] select: initializing btl component self
[cn2:01046] select: init of component self returned success
[cn2:01046] select: init returned priority 20
[cn2:01046] selected ob1 best priority 20
[cn2:01046] select: component ob1 selected
[cn2:01046] mca: base: close: component v closed
[cn2:01046] mca: base: close: unloading component v
[cn2:01046] mca: base: close: component bfo closed
[cn2:01046] mca: base: close: unloading component bfo
[cn2:01046] [[15336,1],3] running modex
[cn2:01046] [[15336,1],3] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:325
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 133 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 596
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 596 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 596 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 209 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:854:udcm_module_create_listen_qp] listening for connections on lid 4, qpn 37963
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:598:udcm_module_init] my modex = LID: 4, Port: 1, QPN: 37963
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:472:udcm_component_query] available for use on mlx4_0:1
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 637 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 812 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],2]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 139
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 764 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1290 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2619
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 216 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2619 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 2619 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 932 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01046] [[15336,1],3] grpcomm:base:modex: performing modex
[cn2:01046] fetching data from PEER dstore
[cn2:01046] dstore:hash:fetch: searching for key NULL on proc 13889961985
[cn2:01046] dstore:hash:fetch: adding data for key MPI_THREAD_LEVEL on proc 13889961985
[cn2:01046] dstore:hash:fetch: adding data for key btl.openib.1.9 on proc 13889961985
[cn2:01046] [[15336,1],3] grpcomm:base:pack_modex: reporting 2 entries
[cn2:01046] [[15336,1],3] grpcomm:base:pack_modex: packing entry for MPI_THREAD_LEVEL
[cn2:01046] [[15336,1],3] grpcomm:base:pack_modex: packing entry for btl.openib.1.9
[cn2:01046] [[15336,1],3] grpcomm:base:full:modex: executing allgather
[cn2:01046] [[15336,1],3] grpcomm:bad entering allgather
[cn2:01046] [[15336,1],3] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:315
[cn2:01046] [[15336,1],3] grpcomm:base:modex: modex posted
[cn2:01046] [[15336,1],3] grpcomm:bad sending collective 0 to our daemon
[cn2:01046] [[15336,1],3] rml_send_buffer to peer [[15336,0],1] at tag 30
[cn2:01046] [[15336,1],3] grpcomm:bad allgather underway
[cn2:01046] [[15336,1],3] rml_send_msg to peer [[15336,0],1] at tag 30
[cn2:01046] [[15336,1],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01046] [[15336,1],3] oob:base:send to target [[15336,0],1]
[cn2:01046] [[15336,1],3] oob:base:send known transport for peer [[15336,0],1]
[cn2:01046] [[15336,1],3] oob:tcp:send_nb to peer [[15336,0],1]:30
[cn2:01046] [[15336,1],3] tcp:send_nb to peer [[15336,0],1]
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:30
[cn2:01046] [[15336,1],3] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_handler called to send to peer [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_handler SENDING TO [[15336,0],1]
[cn2:01046] [[15336,1],3] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 139 BYTES ON SOCKET 11
[cn2:01046] [[15336,1],3]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 596 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 139
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2619 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 194
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 194 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 194 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 194 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:686:init_one_port] looking for mlx4_0:1 GID index 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 219
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 219 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 219 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:717:init_one_port] my IB subnet_id for HCA mlx4_0 port 1 is fe80000000000000
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 79
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 79 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 79 bytes from [[15336,0],3] for tag 2
[cn4:17831] [rank=6] openib: using device mlx4_0
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 219 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1005
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1005 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1005 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1294:setup_qps] pp: rd_num is 256 rd_low is 192 rd_win 128 rd_rsv 4
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:2060:rdmacm_component_query] rdmacm_component_query
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 79 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17831] openib BTL: rdmacm IP address not found on port
[cn4:17831] openib BTL: rdmacm CPC unavailable for use on mlx4_0:1; skipped
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 643
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 643 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 643 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_ip.c:132:mca_btl_openib_rdma_get_ipv4addr] Looking for mlx4_0:1 in IP address list
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:1948:ipaddrcheck] *** Could not find IP address for mlx4_0:1 -- is there an IP address configured for this device?
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:543:udcm_module_init] created cpc module 0xdec3a0 for btl 0xdf7920
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1005 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 643 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 194
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 194 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 194 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 219
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 194 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 219 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:686:init_one_port] looking for mlx4_0:1 GID index 0
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 219 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 216
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:717:init_one_port] my IB subnet_id for HCA mlx4_0 port 1 is fe80000000000000
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 219 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 828
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 828 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 216 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 216 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 828 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 216 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 828 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:793:udcm_module_create_listen_qp] creating listen QP on port 1
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1294:setup_qps] pp: rd_num is 256 rd_low is 192 rd_win 128 rd_rsv 4
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 79
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 79 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 79 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 820
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 820 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] [rank=5] openib: using device mlx4_0
[cn1:14209] [[15336,0],0] message received 820 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:2060:rdmacm_component_query] rdmacm_component_query
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_ip.c:132:mca_btl_openib_rdma_get_ipv4addr] Looking for mlx4_0:1 in IP address list
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:1948:ipaddrcheck] *** Could not find IP address for mlx4_0:1 -- is there an IP address configured for this device?
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:543:udcm_module_init] created cpc module 0x7d4420 for btl 0x7e2120
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 79 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 820 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21420] openib BTL: rdmacm IP address not found on port
[cn3:21420] openib BTL: rdmacm CPC unavailable for use on mlx4_0:1; skipped
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 233
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 233 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 393 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 233 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 233 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 742 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 393
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 393 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:854:udcm_module_create_listen_qp] listening for connections on lid 7, qpn 33170
[cn1:14209] [[15336,0],0] message received 393 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:598:udcm_module_init] my modex = LID: 7, Port: 1, QPN: 33170
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:472:udcm_component_query] available for use on mlx4_0:1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 742
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 742 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 742 bytes from [[15336,0],2] for tag 2
[cn3:21419] storing data in PEER dstore
[cn3:21419] dstore:hash:store storing data for proc 18184929281
[cn3:21419] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 18184929281
[cn3:21419] select: init of component openib returned success
[cn3:21419] select: initializing btl component self
[cn3:21419] select: init of component self returned success
[cn3:21419] select: init returned priority 20
[cn3:21419] selected ob1 best priority 20
[cn3:21419] select: component ob1 selected
[cn3:21419] mca: base: close: component v closed
[cn3:21419] mca: base: close: unloading component v
[cn3:21419] mca: base: close: component bfo closed
[cn3:21419] mca: base: close: unloading component bfo
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 220
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 220 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 220 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 220 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 903 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 903
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 903 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21419] [[15336,1],4] running modex
[cn3:21419] [[15336,1],4] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:325
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 903 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1121
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1121 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1121 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],4]
[cn3:21419] [[15336,1],4] grpcomm:base:modex: performing modex
[cn3:21419] fetching data from PEER dstore
[cn3:21419] dstore:hash:fetch: searching for key NULL on proc 18184929281
[cn3:21419] dstore:hash:fetch: adding data for key MPI_THREAD_LEVEL on proc 18184929281
[cn3:21419] dstore:hash:fetch: adding data for key btl.openib.1.9 on proc 18184929281
[cn3:21419] [[15336,1],4] grpcomm:base:pack_modex: reporting 2 entries
[cn3:21419] [[15336,1],4] grpcomm:base:pack_modex: packing entry for MPI_THREAD_LEVEL
[cn3:21419] [[15336,1],4] grpcomm:base:pack_modex: packing entry for btl.openib.1.9
[cn3:21419] [[15336,1],4] grpcomm:base:full:modex: executing allgather
[cn3:21419] [[15336,1],4] grpcomm:bad entering allgather
[cn3:21419] [[15336,1],4] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:315
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 139
[cn1:14209] [[15336,0],0] message received 1121 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21419] [[15336,1],4] grpcomm:base:modex: modex posted
[cn3:21419] [[15336,1],4] grpcomm:bad sending collective 0 to our daemon
[cn3:21419] [[15336,1],4] rml_send_buffer to peer [[15336,0],2] at tag 30
[cn3:21419] [[15336,1],4] grpcomm:bad allgather underway
[cn3:21419] [[15336,1],4] rml_send_msg to peer [[15336,0],2] at tag 30
[cn3:21419] [[15336,1],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21419] [[15336,1],4] oob:base:send to target [[15336,0],2]
[cn3:21419] [[15336,1],4] oob:base:send known transport for peer [[15336,0],2]
[cn3:21419] [[15336,1],4] oob:tcp:send_nb to peer [[15336,0],2]:30
[cn3:21419] [[15336,1],4] tcp:send_nb to peer [[15336,0],2]
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:30
[cn3:21419] [[15336,1],4] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 655
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 655 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 655 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21419] [[15336,1],4] tcp:send_nb: already connected to [[15336,0],2] - queueing for send
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_handler called to send to peer [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_handler SENDING TO [[15336,0],2]
[cn3:21419] [[15336,1],4] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 139 BYTES ON SOCKET 11
[cn3:21419] [[15336,1],4]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 655 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 216
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 216 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 216 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:793:udcm_module_create_listen_qp] creating listen QP on port 1
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 216 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 596
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 596 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 596 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 596 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:854:udcm_module_create_listen_qp] listening for connections on lid 7, qpn 33171
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:598:udcm_module_init] my modex = LID: 7, Port: 1, QPN: 33171
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:472:udcm_component_query] available for use on mlx4_0:1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 932
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 932 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 932 bytes from [[15336,0],2] for tag 2
[cn3:21420] storing data in PEER dstore
[cn3:21420] dstore:hash:store storing data for proc 22479896577
[cn3:21420] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 22479896577
[cn3:21420] select: init of component openib returned success
[cn3:21420] select: initializing btl component self
[cn3:21420] select: init of component self returned success
[cn3:21420] select: init returned priority 20
[cn3:21420] selected ob1 best priority 20
[cn3:21420] select: component ob1 selected
[cn3:21420] mca: base: close: component v closed
[cn3:21420] mca: base: close: unloading component v
[cn3:21420] mca: base: close: component bfo closed
[cn3:21420] mca: base: close: unloading component bfo
[cn3:21420] [[15336,1],5] running modex
[cn3:21420] [[15336,1],5] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:325
[cn1:14224] mca: base: components_register: registering bml components
[cn1:14224] mca: base: components_register: found loaded component r2
[cn1:14224] mca: base: components_register: component r2 register function successful
[cn1:14224] mca: base: components_open: opening bml components
[cn1:14224] mca: base: components_open: found loaded component r2
[cn1:14224] mca: base: components_open: component r2 open function successful
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2619
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 932 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],5]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 2619 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 139
[cn1:14209] [[15336,0],0] message received 2619 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21420] [[15336,1],5] grpcomm:base:modex: performing modex
[cn3:21420] fetching data from PEER dstore
[cn3:21420] dstore:hash:fetch: searching for key NULL on proc 22479896577
[cn3:21420] dstore:hash:fetch: adding data for key MPI_THREAD_LEVEL on proc 22479896577
[cn3:21420] dstore:hash:fetch: adding data for key btl.openib.1.9 on proc 22479896577
[cn3:21420] [[15336,1],5] grpcomm:base:pack_modex: reporting 2 entries
[cn3:21420] [[15336,1],5] grpcomm:base:pack_modex: packing entry for MPI_THREAD_LEVEL
[cn3:21420] [[15336,1],5] grpcomm:base:pack_modex: packing entry for btl.openib.1.9
[cn3:21420] [[15336,1],5] grpcomm:base:full:modex: executing allgather
[cn3:21420] [[15336,1],5] grpcomm:bad entering allgather
[cn3:21420] [[15336,1],5] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:315
[cn3:21420] [[15336,1],5] grpcomm:base:modex: modex posted
[cn3:21420] [[15336,1],5] grpcomm:bad sending collective 0 to our daemon
[cn3:21420] [[15336,1],5] rml_send_buffer to peer [[15336,0],2] at tag 30
[cn3:21420] [[15336,1],5] grpcomm:bad allgather underway
[cn3:21420] [[15336,1],5] rml_send_msg to peer [[15336,0],2] at tag 30
[cn3:21420] [[15336,1],5] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21420] [[15336,1],5] oob:base:send to target [[15336,0],2]
[cn3:21420] [[15336,1],5] oob:base:send known transport for peer [[15336,0],2]
[cn3:21420] [[15336,1],5] oob:tcp:send_nb to peer [[15336,0],2]:30
[cn3:21420] [[15336,1],5] tcp:send_nb to peer [[15336,0],2]
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:30
[cn3:21420] [[15336,1],5] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_nb: already connected to [[15336,0],2] - queueing for send
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_handler called to send to peer [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_handler SENDING TO [[15336,0],2]
[cn3:21420] [[15336,1],5] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 139 BYTES ON SOCKET 11
[cn3:21420] [[15336,1],5]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2619 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 194
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 194 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 194 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:686:init_one_port] looking for mlx4_0:1 GID index 0
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 194 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 219
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 219 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 219 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 79
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 79 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:717:init_one_port] my IB subnet_id for HCA mlx4_0 port 1 is fe80000000000000
[cn1:14209] [[15336,0],0] message received 79 bytes from [[15336,0],3] for tag 2
[cn4:17832] [rank=7] openib: using device mlx4_0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 828
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 828 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 828 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 820
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1294:setup_qps] pp: rd_num is 256 rd_low is 192 rd_win 128 rd_rsv 4
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 820 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 820 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:2060:rdmacm_component_query] rdmacm_component_query
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_ip.c:132:mca_btl_openib_rdma_get_ipv4addr] Looking for mlx4_0:1 in IP address list
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:1948:ipaddrcheck] *** Could not find IP address for mlx4_0:1 -- is there an IP address configured for this device?
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:543:udcm_module_init] created cpc module 0xd6f3a0 for btl 0xd7a920
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 219 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17832] openib BTL: rdmacm IP address not found on port
[cn4:17832] openib BTL: rdmacm CPC unavailable for use on mlx4_0:1; skipped
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 79 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 828 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 820 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 932
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 932 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 932 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 932 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17831] storing data in PEER dstore
[cn4:17831] dstore:hash:store storing data for proc 26774863873
[cn4:17831] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 26774863873
[cn4:17831] select: init of component openib returned success
[cn4:17831] select: initializing btl component self
[cn4:17831] select: init of component self returned success
[cn4:17831] select: init returned priority 20
[cn4:17831] selected ob1 best priority 20
[cn4:17831] select: component ob1 selected
[cn4:17831] mca: base: close: component v closed
[cn4:17831] mca: base: close: unloading component v
[cn4:17831] mca: base: close: component bfo closed
[cn4:17831] mca: base: close: unloading component bfo
[cn4:17831] [[15336,1],6] running modex
[cn4:17831] [[15336,1],6] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:325
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 596 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 596
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 596 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 596 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 384
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 384 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 384 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:854:udcm_module_create_listen_qp] listening for connections on lid 5, qpn 32327
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:598:udcm_module_init] my modex = LID: 5, Port: 1, QPN: 32327
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:472:udcm_component_query] available for use on mlx4_0:1
[cn1:14209] [[15336,0],0] message received 384 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17831] [[15336,1],6] grpcomm:base:modex: performing modex
[cn4:17831] fetching data from PEER dstore
[cn4:17831] dstore:hash:fetch: searching for key NULL on proc 26774863873
[cn4:17831] dstore:hash:fetch: adding data for key MPI_THREAD_LEVEL on proc 26774863873
[cn4:17831] dstore:hash:fetch: adding data for key btl.openib.1.9 on proc 26774863873
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1871
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],6]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 139
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1871 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1871 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1871 bytes from [[15336,0],3] for tag 2
[cn4:17831] [[15336,1],6] grpcomm:base:pack_modex: reporting 2 entries
[cn4:17831] [[15336,1],6] grpcomm:base:pack_modex: packing entry for MPI_THREAD_LEVEL
[cn4:17831] [[15336,1],6] grpcomm:base:pack_modex: packing entry for btl.openib.1.9
[cn4:17831] [[15336,1],6] grpcomm:base:full:modex: executing allgather
[cn4:17831] [[15336,1],6] grpcomm:bad entering allgather
[cn4:17831] [[15336,1],6] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:315
[cn4:17831] [[15336,1],6] grpcomm:base:modex: modex posted
[cn4:17831] [[15336,1],6] grpcomm:bad sending collective 0 to our daemon
[cn4:17831] [[15336,1],6] rml_send_buffer to peer [[15336,0],3] at tag 30
[cn4:17831] [[15336,1],6] grpcomm:bad allgather underway
[cn4:17831] [[15336,1],6] rml_send_msg to peer [[15336,0],3] at tag 30
[cn4:17831] [[15336,1],6] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17831] [[15336,1],6] oob:base:send to target [[15336,0],3]
[cn4:17831] [[15336,1],6] oob:base:send known transport for peer [[15336,0],3]
[cn4:17831] [[15336,1],6] oob:tcp:send_nb to peer [[15336,0],3]:30
[cn4:17831] [[15336,1],6] tcp:send_nb to peer [[15336,0],3]
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:30
[cn4:17831] [[15336,1],6] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn4:17831] [[15336,1],6] tcp:send_nb: already connected to [[15336,0],3] - queueing for send
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 424
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 424 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 424 bytes from [[15336,0],3] for tag 2
[cn4:17831] [[15336,1],6] tcp:send_handler called to send to peer [[15336,0],3]
[cn4:17831] [[15336,1],6] tcp:send_handler SENDING TO [[15336,0],3]
[cn4:17831] [[15336,1],6] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 139 BYTES ON SOCKET 11
[cn4:17831] [[15336,1],6]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14224] mca: base: components_register: registering btl components
[cn1:14224] mca: base: components_register: found loaded component openib
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 216
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 216 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 216 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:793:udcm_module_create_listen_qp] creating listen QP on port 1
[cn1:14224] mca: base: components_register: component openib register function successful
[cn1:14224] mca: base: components_register: found loaded component self
[cn1:14224] mca: base: components_register: component self register function successful
[cn1:14224] mca: base: components_open: opening btl components
[cn1:14224] mca: base: components_open: found loaded component openib
[cn1:14224] mca: base: components_open: component openib open function successful
[cn1:14224] mca: base: components_open: found loaded component self
[cn1:14224] mca: base: components_open: component self open function successful
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 424 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 216 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 596
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 596 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 596 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 932
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 932 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 596 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:854:udcm_module_create_listen_qp] listening for connections on lid 5, qpn 32328
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:598:udcm_module_init] my modex = LID: 5, Port: 1, QPN: 32328
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:472:udcm_component_query] available for use on mlx4_0:1
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 932 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 932 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17832] storing data in PEER dstore
[cn4:17832] dstore:hash:store storing data for proc 31069831169
[cn4:17832] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 31069831169
[cn4:17832] select: init of component openib returned success
[cn4:17832] select: initializing btl component self
[cn4:17832] select: init of component self returned success
[cn4:17832] select: init returned priority 20
[cn4:17832] selected ob1 best priority 20
[cn4:17832] select: component ob1 selected
[cn4:17832] mca: base: close: component v closed
[cn4:17832] mca: base: close: unloading component v
[cn4:17832] mca: base: close: component bfo closed
[cn4:17832] mca: base: close: unloading component bfo
[cn4:17832] [[15336,1],7] running modex
[cn4:17832] [[15336,1],7] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:325
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2619
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],7]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 2619 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 139
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 2619 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17832] [[15336,1],7] grpcomm:base:modex: performing modex
[cn4:17832] fetching data from PEER dstore
[cn4:17832] dstore:hash:fetch: searching for key NULL on proc 31069831169
[cn4:17832] dstore:hash:fetch: adding data for key MPI_THREAD_LEVEL on proc 31069831169
[cn4:17832] dstore:hash:fetch: adding data for key btl.openib.1.9 on proc 31069831169
[cn4:17832] [[15336,1],7] grpcomm:base:pack_modex: reporting 2 entries
[cn4:17832] [[15336,1],7] grpcomm:base:pack_modex: packing entry for MPI_THREAD_LEVEL
[cn4:17832] [[15336,1],7] grpcomm:base:pack_modex: packing entry for btl.openib.1.9
[cn4:17832] [[15336,1],7] grpcomm:base:full:modex: executing allgather
[cn4:17832] [[15336,1],7] grpcomm:bad entering allgather
[cn4:17832] [[15336,1],7] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:315
[cn4:17832] [[15336,1],7] grpcomm:base:modex: modex posted
[cn4:17832] [[15336,1],7] grpcomm:bad sending collective 0 to our daemon
[cn4:17832] [[15336,1],7] rml_send_buffer to peer [[15336,0],3] at tag 30
[cn4:17832] [[15336,1],7] grpcomm:bad allgather underway
[cn4:17832] [[15336,1],7] rml_send_msg to peer [[15336,0],3] at tag 30
[cn4:17832] [[15336,1],7] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17832] [[15336,1],7] oob:base:send to target [[15336,0],3]
[cn4:17832] [[15336,1],7] oob:base:send known transport for peer [[15336,0],3]
[cn4:17832] [[15336,1],7] oob:tcp:send_nb to peer [[15336,0],3]:30
[cn4:17832] [[15336,1],7] tcp:send_nb to peer [[15336,0],3]
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:30
[cn4:17832] [[15336,1],7] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_nb: already connected to [[15336,0],3] - queueing for send
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_handler called to send to peer [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_handler SENDING TO [[15336,0],3]
[cn4:17832] [[15336,1],7] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 139 BYTES ON SOCKET 11
[cn4:17832] [[15336,1],7]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2619 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14224] mca: base: components_register: registering pml components
[cn1:14224] mca: base: components_register: found loaded component v
[cn1:14224] mca: base: components_register: component v register function successful
[cn1:14224] mca: base: components_register: found loaded component bfo
[cn1:14224] mca: base: components_register: component bfo register function successful
[cn1:14224] mca: base: components_register: found loaded component cm
[cn1:14224] mca: base: components_register: component cm register function successful
[cn1:14224] mca: base: components_register: found loaded component ob1
[cn1:14224] mca: base: components_register: component ob1 register function successful
[cn1:14224] mca: base: components_open: opening pml components
[cn1:14224] mca: base: components_open: found loaded component v
[cn1:14224] mca: base: components_open: component v open function successful
[cn1:14224] mca: base: components_open: found loaded component bfo
[cn1:14224] mca: base: components_open: component bfo open function successful
[cn1:14224] mca: base: components_open: found loaded component cm
[cn1:14225] [[15336,1],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14225] [[15336,1],1]:tcp:recv:handler CONNECTED
[cn1:14225] [[15336,1],1] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 14140 BYTES FOR DEST [[15336,1],1] TAG 20
[cn1:14225] [[15336,1],1] DELIVERING TO RML
[cn1:14225] [[15336,1],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14225] [[15336,1],1] message received 14140 bytes from [[15336,0],0] for tag 20
[cn1:14225] [[15336,1],1] registering sync ack recvd
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 1005060096
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060096
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18446744069414584318
[cn1:14225] dstore:hash:store: storing key cn1[OPAL_UINT32] for proc 18446744069414584318
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027393
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 5300027393
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027392
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 5300027392
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18446744069414584318
[cn1:14225] dstore:hash:store: storing key cn2[OPAL_UINT32] for proc 18446744069414584318
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 9594994688
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994688
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18446744069414584318
[cn1:14225] dstore:hash:store: storing key cn3[OPAL_UINT32] for proc 18446744069414584318
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 13889961984
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961984
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18446744069414584318
[cn1:14225] dstore:hash:store: storing key cn4[OPAL_UINT32] for proc 18446744069414584318
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18184929280
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929280
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18446744069414584318
[cn1:14225] dstore:hash:store: storing key cn5[OPAL_UINT32] for proc 18446744069414584318
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18446744070419644417
[cn1:14225] dstore:hash:store: storing key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18446744070419644417
[cn1:14225] dstore:hash:store: updating key orte.nprocs[OPAL_UINT32] for proc 18446744070419644417
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key opal.local.ldr[OPAL_UINT64] for proc 5300027393
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 1005060097
[cn1:14225] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 1005060097
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 1005060097
[cn1:14225] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 1005060097
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 1005060097
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 1005060097
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 1005060097
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 1005060097
[cn1:14225] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 1005060097
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 1005060096
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 1005060097
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 1005060097
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 1005060097
[cn1:14225] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 1005060097
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 5300027393
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 5300027393
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 5300027393
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 9594994689
[cn1:14225] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 9594994689
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 9594994689
[cn1:14225] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 9594994689
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 9594994689
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 9594994689
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 9594994689
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 9594994689
[cn1:14225] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 9594994689
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 9594994689
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 9594994689
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 9594994689
[cn1:14225] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 9594994689
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 13889961985
[cn1:14225] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 13889961985
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 13889961985
[cn1:14225] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 13889961985
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 13889961985
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 13889961985
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 13889961985
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 13889961985
[cn1:14225] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 13889961985
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 5300027392
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 13889961985
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 13889961985
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 13889961985
[cn1:14225] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 13889961985
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18184929281
[cn1:14225] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 18184929281
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18184929281
[cn1:14225] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 18184929281
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18184929281
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 18184929281
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 18184929281
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 18184929281
[cn1:14225] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 18184929281
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 18184929281
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 18184929281
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18184929281
[cn1:14225] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 18184929281
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 22479896577
[cn1:14225] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 22479896577
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 22479896577
[cn1:14225] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 22479896577
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 22479896577
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 22479896577
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 22479896577
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 22479896577
[cn1:14225] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 22479896577
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 9594994688
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 22479896577
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 22479896577
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 22479896577
[cn1:14225] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 22479896577
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 26774863873
[cn1:14225] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 26774863873
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 26774863873
[cn1:14225] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 26774863873
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 26774863873
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 26774863873
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 26774863873
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 26774863873
[cn1:14225] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 26774863873
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 26774863873
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 26774863873
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 26774863873
[cn1:14225] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 26774863873
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 31069831169
[cn1:14225] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 31069831169
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 31069831169
[cn1:14225] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 31069831169
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 31069831169
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 31069831169
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 31069831169
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 31069831169
[cn1:14225] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 31069831169
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 13889961984
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 31069831169
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 31069831169
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 31069831169
[cn1:14225] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 31069831169
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 35364798465
[cn1:14225] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 35364798465
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 35364798465
[cn1:14225] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 35364798465
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 35364798465
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 35364798465
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 35364798465
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 35364798465
[cn1:14225] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 35364798465
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 35364798465
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 35364798465
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 35364798465
[cn1:14225] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 35364798465
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 39659765761
[cn1:14225] dstore:hash:store: storing key opal.local.rank[OPAL_UINT16] for proc 39659765761
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 39659765761
[cn1:14225] dstore:hash:store: storing key orte.node.rank[OPAL_UINT16] for proc 39659765761
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 39659765761
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 39659765761
[cn1:14225] dstore:hash:store: storing key opal.cpuset[OPAL_STRING] for proc 39659765761
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 39659765761
[cn1:14225] dstore:hash:store: storing key orte.daemon.vpid[OPAL_UINT32] for proc 39659765761
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 18184929280
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 39659765761
[cn1:14225] dstore:hash:store: storing key orte.hostname[OPAL_STRING] for proc 39659765761
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 39659765761
[cn1:14225] dstore:hash:store: storing key orte.global.rank[OPAL_UINT32] for proc 39659765761
[cn1:14225] [[15336,1],1] Process bound at launch
[cn1:14225] MCW rank 1 bound to socket 0[core 1[hwt 0]]: [./B/./.][./././.]
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: updating key opal.cpuset[OPAL_STRING] for proc 5300027393
[cn1:14225] storing data in PEER dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 5300027393
[cn1:14225] mca: base: components_register: registering bml components
[cn1:14225] mca: base: components_register: found loaded component r2
[cn1:14225] mca: base: components_register: component r2 register function successful
[cn1:14225] mca: base: components_open: opening bml components
[cn1:14225] mca: base: components_open: found loaded component r2
[cn1:14225] mca: base: components_open: component r2 open function successful
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],8]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,1],8] (ORIGIN [[15336,1],8]) OF 139 BYTES FOR DEST [[15336,0],4] TAG 30
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] message received 139 bytes from [[15336,1],8] for tag 30
[cn5:14171] [[15336,0],4] COLLECTIVE RECVD FROM [[15336,1],8]
[cn5:14171] [[15336,0],4] WORKING COLLECTIVE 0
[cn5:14171] [[15336,0],4] PROGRESSING COLLECTIVE 0
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 0
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 1
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 2
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14225] mca: base: components_register: registering btl components
[cn1:14225] mca: base: components_register: found loaded component openib
[cn1:14225] mca: base: components_register: component openib register function successful
[cn1:14225] mca: base: components_register: found loaded component self
[cn1:14225] mca: base: components_register: component self register function successful
[cn1:14225] mca: base: components_open: opening btl components
[cn1:14225] mca: base: components_open: found loaded component openib
[cn1:14225] mca: base: components_open: component openib open function successful
[cn1:14225] mca: base: components_open: found loaded component self
[cn1:14225] mca: base: components_open: component self open function successful
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],9]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,1],9] (ORIGIN [[15336,1],9]) OF 139 BYTES FOR DEST [[15336,0],4] TAG 30
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] message received 139 bytes from [[15336,1],9] for tag 30
[cn5:14171] [[15336,0],4] COLLECTIVE RECVD FROM [[15336,1],9]
[cn5:14171] [[15336,0],4] WORKING COLLECTIVE 0
[cn5:14171] [[15336,0],4] PROGRESSING COLLECTIVE 0
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 0
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn5:14171] [[15336,0],4] COLLECTIVE 0 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],4] at tag 32
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 2
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],4] at tag 32
[cn5:14171] [[15336,0],4] rml_send_iovec_to_self at tag 32
[cn5:14171] [[15336,0],4] rml_send_to_self callback executing for tag 32
[cn5:14171] [[15336,0],4] message received 290 bytes from [[15336,0],4] for tag 32
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],4]
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: WORKING COLLECTIVE 0
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 32
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 290
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 290 BYTES FOR DEST [[15336,0],0] TAG 32
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 290 bytes from [[15336,0],4] for tag 32
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:32
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:32
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 290 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 0
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: CANNOT PROGRESS
[cn1:14225] mca: base: components_register: registering pml components
[cn1:14225] mca: base: components_register: found loaded component v
[cn1:14225] mca: base: components_register: component v register function successful
[cn1:14225] mca: base: components_register: found loaded component bfo
[cn1:14225] mca: base: components_register: component bfo register function successful
[cn1:14225] mca: base: components_register: found loaded component cm
[cn1:14225] mca: base: components_register: component cm register function successful
[cn1:14225] mca: base: components_register: found loaded component ob1
[cn1:14225] mca: base: components_register: component ob1 register function successful
[cn1:14225] mca: base: components_open: opening pml components
[cn1:14225] mca: base: components_open: found loaded component v
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],2]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn1:14225] mca: base: components_open: component v open function successful
[cn1:14225] mca: base: components_open: found loaded component bfo
[cn1:14225] mca: base: components_open: component bfo open function successful
[cn1:14225] mca: base: components_open: found loaded component cm
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,1],2] (ORIGIN [[15336,1],2]) OF 139 BYTES FOR DEST [[15336,0],1] TAG 30
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] message received 139 bytes from [[15336,1],2] for tag 30
[cn2:01042] [[15336,0],1] COLLECTIVE RECVD FROM [[15336,1],2]
[cn2:01042] [[15336,0],1] WORKING COLLECTIVE 0
[cn2:01042] [[15336,0],1] PROGRESSING COLLECTIVE 0
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 0
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 1
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 2
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,1],3] (ORIGIN [[15336,1],3]) OF 139 BYTES FOR DEST [[15336,0],1] TAG 30
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] message received 139 bytes from [[15336,1],3] for tag 30
[cn2:01042] [[15336,0],1] COLLECTIVE RECVD FROM [[15336,1],3]
[cn2:01042] [[15336,0],1] WORKING COLLECTIVE 0
[cn2:01042] [[15336,0],1] PROGRESSING COLLECTIVE 0
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 0
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] COLLECTIVE 0 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],1] at tag 32
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 1
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 2
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],1] at tag 32
[cn2:01042] [[15336,0],1] rml_send_iovec_to_self at tag 32
[cn2:01042] [[15336,0],1] rml_send_to_self callback executing for tag 32
[cn2:01042] [[15336,0],1] message received 290 bytes from [[15336,0],1] for tag 32
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],1]
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: WORKING COLLECTIVE 0
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],4]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,1],4] (ORIGIN [[15336,1],4]) OF 139 BYTES FOR DEST [[15336,0],2] TAG 30
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] message received 139 bytes from [[15336,1],4] for tag 30
[cn3:21416] [[15336,0],2] COLLECTIVE RECVD FROM [[15336,1],4]
[cn3:21416] [[15336,0],2] WORKING COLLECTIVE 0
[cn3:21416] [[15336,0],2] PROGRESSING COLLECTIVE 0
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 0
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 1
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 2
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],5]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,1],5] (ORIGIN [[15336,1],5]) OF 139 BYTES FOR DEST [[15336,0],2] TAG 30
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] message received 139 bytes from [[15336,1],5] for tag 30
[cn3:21416] [[15336,0],2] COLLECTIVE RECVD FROM [[15336,1],5]
[cn3:21416] [[15336,0],2] WORKING COLLECTIVE 0
[cn3:21416] [[15336,0],2] PROGRESSING COLLECTIVE 0
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 0
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] COLLECTIVE 0 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],2] at tag 32
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 1
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 2
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],2] at tag 32
[cn3:21416] [[15336,0],2] rml_send_iovec_to_self at tag 32
[cn3:21416] [[15336,0],2] rml_send_to_self callback executing for tag 32
[cn3:21416] [[15336,0],2] message received 290 bytes from [[15336,0],2] for tag 32
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],2]
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: WORKING COLLECTIVE 0
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 32
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:32
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:32
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 290 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 290
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 290 BYTES FOR DEST [[15336,0],0] TAG 32
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 290 bytes from [[15336,0],2] for tag 32
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],2]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 0
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: CANNOT PROGRESS
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],6]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,1],6] (ORIGIN [[15336,1],6]) OF 139 BYTES FOR DEST [[15336,0],3] TAG 30
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] message received 139 bytes from [[15336,1],6] for tag 30
[cn4:17828] [[15336,0],3] COLLECTIVE RECVD FROM [[15336,1],6]
[cn4:17828] [[15336,0],3] WORKING COLLECTIVE 0
[cn4:17828] [[15336,0],3] PROGRESSING COLLECTIVE 0
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 0
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 1
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 2
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],7]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,1],7] (ORIGIN [[15336,1],7]) OF 139 BYTES FOR DEST [[15336,0],3] TAG 30
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] message received 139 bytes from [[15336,1],7] for tag 30
[cn4:17828] [[15336,0],3] COLLECTIVE RECVD FROM [[15336,1],7]
[cn4:17828] [[15336,0],3] WORKING COLLECTIVE 0
[cn4:17828] [[15336,0],3] PROGRESSING COLLECTIVE 0
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 0
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] COLLECTIVE 0 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],3] at tag 32
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 1
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 2
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],3] at tag 32
[cn4:17828] [[15336,0],3] rml_send_iovec_to_self at tag 32
[cn4:17828] [[15336,0],3] rml_send_to_self callback executing for tag 32
[cn4:17828] [[15336,0],3] message received 290 bytes from [[15336,0],3] for tag 32
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],3]
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: WORKING COLLECTIVE 0
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],1]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],1] at tag 32
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],1] at tag 32
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 290
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],1]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],1]:32
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:32
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],1]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 290 BYTES ON SOCKET 10
[cn4:17828] [[15336,0],3]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 290 BYTES FOR DEST [[15336,0],1] TAG 32
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] message received 290 bytes from [[15336,0],3] for tag 32
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],3]
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: WORKING COLLECTIVE 0
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 4
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 32
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:32
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:32
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 548
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 548 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 548 BYTES FOR DEST [[15336,0],0] TAG 32
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 548 bytes from [[15336,0],1] for tag 32
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],1]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 0
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 4
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: CANNOT PROGRESS
[cn1:14224] mca: base: components_register: registering mtl components
[cn1:14224] mca: base: components_register: found loaded component psm
[cn1:14224] mca: base: components_register: component psm register function successful
[cn1:14224] mca: base: components_open: opening mtl components
[cn1:14224] mca: base: components_open: found loaded component psm
[cn1:14224] mca: base: close: component psm closed
[cn1:14224] mca: base: close: unloading component psm
[cn1:14224] mca: base: close: component cm closed
[cn1:14224] mca: base: close: unloading component cm
[cn1:14224] mca: base: components_open: found loaded component ob1
[cn1:14224] mca: base: components_open: component ob1 open function successful
[cn1:14224] select: component v not in the include list
[cn1:14224] select: component bfo not in the include list
[cn1:14224] select: initializing pml component ob1
[cn1:14224] select: initializing btl component openib
[cn1:14225] mca: base: components_register: registering mtl components
[cn1:14225] mca: base: components_register: found loaded component psm
[cn1:14225] mca: base: components_register: component psm register function successful
[cn1:14225] mca: base: components_open: opening mtl components
[cn1:14225] mca: base: components_open: found loaded component psm
[cn1:14225] mca: base: close: component psm closed
[cn1:14225] mca: base: close: unloading component psm
[cn1:14225] mca: base: close: component cm closed
[cn1:14225] mca: base: close: unloading component cm
[cn1:14225] mca: base: components_open: found loaded component ob1
[cn1:14225] mca: base: components_open: component ob1 open function successful
[cn1:14225] select: component v not in the include list
[cn1:14225] select: component bfo not in the include list
[cn1:14225] select: initializing pml component ob1
[cn1:14225] select: initializing btl component openib
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:686:init_one_port] looking for mlx4_0:1 GID index 0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:717:init_one_port] my IB subnet_id for HCA mlx4_0 port 1 is fe80000000000000
[cn1:14224] [rank=0] openib: using device mlx4_0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1294:setup_qps] pp: rd_num is 256 rd_low is 192 rd_win 128 rd_rsv 4
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:2060:rdmacm_component_query] rdmacm_component_query
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_ip.c:132:mca_btl_openib_rdma_get_ipv4addr] Looking for mlx4_0:1 in IP address list
[cn1:14224] openib BTL: rdmacm IP address not found on port
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:1948:ipaddrcheck] *** Could not find IP address for mlx4_0:1 -- is there an IP address configured for this device?
[cn1:14224] openib BTL: rdmacm CPC unavailable for use on mlx4_0:1; skipped
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:543:udcm_module_init] created cpc module 0xe349c0 for btl 0xe3fda0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:793:udcm_module_create_listen_qp] creating listen QP on port 1
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:686:init_one_port] looking for mlx4_0:1 GID index 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 139
[cn1:14225] [rank=1] openib: using device mlx4_0
[cn1:14225] openib BTL: rdmacm IP address not found on port
[cn1:14225] openib BTL: rdmacm CPC unavailable for use on mlx4_0:1; skipped
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:854:udcm_module_create_listen_qp] listening for connections on lid 3, qpn 37181
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:598:udcm_module_init] my modex = LID: 3, Port: 1, QPN: 37181
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:472:udcm_component_query] available for use on mlx4_0:1
[cn1:14224] storing data in PEER dstore
[cn1:14224] dstore:hash:store storing data for proc 1005060097
[cn1:14224] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 1005060097
[cn1:14224] select: init of component openib returned success
[cn1:14224] select: initializing btl component self
[cn1:14224] select: init of component self returned success
[cn1:14224] select: init returned priority 20
[cn1:14224] selected ob1 best priority 20
[cn1:14224] select: component ob1 selected
[cn1:14224] mca: base: close: component v closed
[cn1:14224] mca: base: close: unloading component v
[cn1:14224] mca: base: close: component bfo closed
[cn1:14224] mca: base: close: unloading component bfo
[cn1:14224] [[15336,1],0] running modex
[cn1:14224] [[15336,1],0] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:325
[cn1:14224] [[15336,1],0] grpcomm:base:modex: performing modex
[cn1:14224] fetching data from PEER dstore
[cn1:14224] dstore:hash:fetch: searching for key NULL on proc 1005060097
[cn1:14224] dstore:hash:fetch: adding data for key MPI_THREAD_LEVEL on proc 1005060097
[cn1:14224] dstore:hash:fetch: adding data for key btl.openib.1.9 on proc 1005060097
[cn1:14224] [[15336,1],0] grpcomm:base:pack_modex: reporting 2 entries
[cn1:14224] [[15336,1],0] grpcomm:base:pack_modex: packing entry for MPI_THREAD_LEVEL
[cn1:14224] [[15336,1],0] grpcomm:base:pack_modex: packing entry for btl.openib.1.9
[cn1:14224] [[15336,1],0] grpcomm:base:full:modex: executing allgather
[cn1:14224] [[15336,1],0] grpcomm:bad entering allgather
[cn1:14224] [[15336,1],0] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:315
[cn1:14224] [[15336,1],0] grpcomm:base:modex: modex posted
[cn1:14224] [[15336,1],0] grpcomm:bad sending collective 0 to our daemon
[cn1:14224] [[15336,1],0] rml_send_buffer to peer [[15336,0],0] at tag 30
[cn1:14224] [[15336,1],0] grpcomm:bad allgather underway
[cn1:14224] [[15336,1],0] rml_send_msg to peer [[15336,0],0] at tag 30
[cn1:14224] [[15336,1],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14224] [[15336,1],0] oob:base:send to target [[15336,0],0]
[cn1:14224] [[15336,1],0] oob:base:send known transport for peer [[15336,0],0]
[cn1:14224] [[15336,1],0] oob:tcp:send_nb to peer [[15336,0],0]:30
[cn1:14224] [[15336,1],0] tcp:send_nb to peer [[15336,0],0]
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:30
[cn1:14224] [[15336,1],0] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14224] [[15336,1],0] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 139 BYTES ON SOCKET 11
[cn1:14224] [[15336,1],0]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:717:init_one_port] my IB subnet_id for HCA mlx4_0 port 1 is fe80000000000000
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1294:setup_qps] pp: rd_num is 256 rd_low is 192 rd_win 128 rd_rsv 4
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:1339:setup_qps] srq: rd_num is 1024 rd_low is 1008 sd_max is 64 rd_max is 256 srq_limit is 48
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:2060:rdmacm_component_query] rdmacm_component_query
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_ip.c:132:mca_btl_openib_rdma_get_ipv4addr] Looking for mlx4_0:1 in IP address list
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c:1948:ipaddrcheck] *** Could not find IP address for mlx4_0:1 -- is there an IP address configured for this device?
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:543:udcm_module_init] created cpc module 0x144fa20 for btl 0x145ad60
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:793:udcm_module_create_listen_qp] creating listen QP on port 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 139
[cn1:14225] storing data in PEER dstore
[cn1:14225] dstore:hash:store storing data for proc 5300027393
[cn1:14225] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 5300027393
[cn1:14225] select: init of component openib returned success
[cn1:14225] select: initializing btl component self
[cn1:14225] select: init of component self returned success
[cn1:14225] select: init returned priority 20
[cn1:14225] selected ob1 best priority 20
[cn1:14225] select: component ob1 selected
[cn1:14225] mca: base: close: component v closed
[cn1:14225] mca: base: close: unloading component v
[cn1:14225] mca: base: close: component bfo closed
[cn1:14225] mca: base: close: unloading component bfo
[cn1:14225] [[15336,1],1] running modex
[cn1:14225] [[15336,1],1] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:325
[cn1:14225] [[15336,1],1] grpcomm:base:modex: performing modex
[cn1:14225] fetching data from PEER dstore
[cn1:14225] dstore:hash:fetch: searching for key NULL on proc 5300027393
[cn1:14225] dstore:hash:fetch: adding data for key MPI_THREAD_LEVEL on proc 5300027393
[cn1:14225] dstore:hash:fetch: adding data for key btl.openib.1.9 on proc 5300027393
[cn1:14225] [[15336,1],1] grpcomm:base:pack_modex: reporting 2 entries
[cn1:14225] [[15336,1],1] grpcomm:base:pack_modex: packing entry for MPI_THREAD_LEVEL
[cn1:14225] [[15336,1],1] grpcomm:base:pack_modex: packing entry for btl.openib.1.9
[cn1:14225] [[15336,1],1] grpcomm:base:full:modex: executing allgather
[cn1:14225] [[15336,1],1] grpcomm:bad entering allgather
[cn1:14225] [[15336,1],1] ACTIVATING GRCPCOMM OP 0 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:315
[cn1:14225] [[15336,1],1] grpcomm:base:modex: modex posted
[cn1:14225] [[15336,1],1] grpcomm:bad sending collective 0 to our daemon
[cn1:14225] [[15336,1],1] rml_send_buffer to peer [[15336,0],0] at tag 30
[cn1:14225] [[15336,1],1] grpcomm:bad allgather underway
[cn1:14225] [[15336,1],1] rml_send_msg to peer [[15336,0],0] at tag 30
[cn1:14225] [[15336,1],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14225] [[15336,1],1] oob:base:send to target [[15336,0],0]
[cn1:14225] [[15336,1],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14225] [[15336,1],1] oob:tcp:send_nb to peer [[15336,0],0]:30
[cn1:14225] [[15336,1],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:30
[cn1:14225] [[15336,1],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14225] [[15336,1],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 139 BYTES ON SOCKET 11
[cn1:14225] [[15336,1],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:854:udcm_module_create_listen_qp] listening for connections on lid 3, qpn 37182
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:598:udcm_module_init] my modex = LID: 3, Port: 1, QPN: 37182
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:472:udcm_component_query] available for use on mlx4_0:1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,1],0] (ORIGIN [[15336,1],0]) OF 139 BYTES FOR DEST [[15336,0],0] TAG 30
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 139 bytes from [[15336,1],0] for tag 30
[cn1:14209] [[15336,0],0] COLLECTIVE RECVD FROM [[15336,1],0]
[cn1:14209] [[15336,0],0] WORKING COLLECTIVE 0
[cn1:14209] [[15336,0],0] PROGRESSING COLLECTIVE 0
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 0
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 1
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 2
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,1],1] (ORIGIN [[15336,1],1]) OF 139 BYTES FOR DEST [[15336,0],0] TAG 30
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 139 bytes from [[15336,1],1] for tag 30
[cn1:14209] [[15336,0],0] COLLECTIVE RECVD FROM [[15336,1],1]
[cn1:14209] [[15336,0],0] WORKING COLLECTIVE 0
[cn1:14209] [[15336,0],0] PROGRESSING COLLECTIVE 0
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 0
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] COLLECTIVE 0 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 1
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 2
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 32
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 32
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 32
[cn1:14209] [[15336,0],0] message received 290 bytes from [[15336,0],0] for tag 32
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],0]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 0
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn1:14209] [[15336,0],0] grpcomm:bad:xcast sent to job [15336,1] tag 30
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 15
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 15
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 15
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 15
[cn1:14209] [[15336,0],0] message received 1330 bytes from [[15336,0],0] for tag 15
[cn1:14209] [[15336,0],0] grpcomm:xcast:recv: with 1330 bytes
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],1]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],1] at tag 15
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],2]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],2] at tag 15
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],4] at tag 15
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 1
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],1] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],2] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],4] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 1
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 1
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],1]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],2]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],4]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],4]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],4]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 1
[cn1:14209] [[15336,0],0] message received 1330 bytes from [[15336,0],0] for tag 1
[cn1:14209] [[15336,0],0] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn1:14209] [[15336,0],0] orted_cmd: received message_local_procs
[cn1:14209] [[15336,0],0] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,1],0] at tag 30
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,1],1] at tag 30
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],2] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],2]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],4] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,1],0] at tag 30
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,1],1] at tag 30
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,1],0]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,1],0]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,1],0]:30
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],0]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,1],1]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,1],1]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,1],1]:30
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],0]:30
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,1],0]) --> [[15336,1],0]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,1],0] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],0]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],1]:30
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,1],1]) --> [[15336,1],1]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,1],1] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],1]
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],1]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],1]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 1330 BYTES ON SOCKET 14
[cn1:14209] [[15336,0],0]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],4]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 1330 BYTES ON SOCKET 15
[cn1:14209] [[15336,0],0]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],2]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],2]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 1330 BYTES ON SOCKET 17
[cn1:14209] [[15336,0],0]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,1],0]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,1],0]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,1],0] OF 1300 BYTES ON SOCKET 20
[cn1:14209] [[15336,0],0]-[[15336,1],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,1],1]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,1],1]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,1],1] OF 1300 BYTES ON SOCKET 0
[cn1:14209] [[15336,0],0]-[[15336,1],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14224] [[15336,1],0]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14224] [[15336,1],0]:tcp:recv:handler CONNECTED
[cn1:14224] [[15336,1],0]:tcp:recv:handler allocate new recv msg
[cn1:14224] [[15336,1],0]:tcp:recv:handler read hdr
[cn1:14224] [[15336,1],0]:tcp:recv:handler allocate data region of size 1300
[cn1:14224] [[15336,1],0] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 1300 BYTES FOR DEST [[15336,1],0] TAG 30
[cn1:14224] [[15336,1],0] DELIVERING TO RML
[cn1:14224] [[15336,1],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14224] [[15336,1],0] message received 1300 bytes from [[15336,0],0] for tag 30
[cn1:14224] [[15336,1],0] grpcomm:base:receive processing collective return for id 0 recvd from [[15336,0],0]
[cn1:14224] [[15336,1],0] CHECKING COLL id 0
[cn1:14224] [[15336,1],0] STORING MODEX DATA
[cn1:14224] [[15336,1],0] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],8]
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],8] FOR MPI_THREAD_LEVEL
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 35364798465
[cn1:14224] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 35364798465
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],8] FOR btl.openib.1.9
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 35364798465
[cn1:14224] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 35364798465
[cn1:14224] [[15336,1],0] store:peer:modex: completed modex entry for proc [[15336,1],8]
[cn1:14224] [[15336,1],0] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],9]
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],9] FOR MPI_THREAD_LEVEL
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 39659765761
[cn1:14224] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 39659765761
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],9] FOR btl.openib.1.9
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 39659765761
[cn1:14224] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 39659765761
[cn1:14224] [[15336,1],0] store:peer:modex: completed modex entry for proc [[15336,1],9]
[cn1:14224] [[15336,1],0] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],4]
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],4] FOR MPI_THREAD_LEVEL
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 18184929281
[cn1:14224] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 18184929281
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],4] FOR btl.openib.1.9
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 18184929281
[cn1:14224] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 18184929281
[cn1:14224] [[15336,1],0] store:peer:modex: completed modex entry for proc [[15336,1],4]
[cn1:14224] [[15336,1],0] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],5]
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],5] FOR MPI_THREAD_LEVEL
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 22479896577
[cn1:14224] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 22479896577
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],5] FOR btl.openib.1.9
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 22479896577
[cn1:14224] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 22479896577
[cn1:14224] [[15336,1],0] store:peer:modex: completed modex entry for proc [[15336,1],5]
[cn1:14224] [[15336,1],0] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],2]
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],2] FOR MPI_THREAD_LEVEL
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 9594994689
[cn1:14224] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 9594994689
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],2] FOR btl.openib.1.9
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 9594994689
[cn1:14224] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 9594994689
[cn1:14224] [[15336,1],0] store:peer:modex: completed modex entry for proc [[15336,1],2]
[cn1:14224] [[15336,1],0] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],3]
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],3] FOR MPI_THREAD_LEVEL
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 13889961985
[cn1:14224] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 13889961985
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],3] FOR btl.openib.1.9
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 13889961985
[cn1:14224] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 13889961985
[cn1:14224] [[15336,1],0] store:peer:modex: completed modex entry for proc [[15336,1],3]
[cn1:14224] [[15336,1],0] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],6]
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],6] FOR MPI_THREAD_LEVEL
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 26774863873
[cn1:14224] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 26774863873
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],6] FOR btl.openib.1.9
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 26774863873
[cn1:14224] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 26774863873
[cn1:14224] [[15336,1],0] store:peer:modex: completed modex entry for proc [[15336,1],6]
[cn1:14224] [[15336,1],0] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],7]
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],7] FOR MPI_THREAD_LEVEL
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 31069831169
[cn1:14224] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 31069831169
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],7] FOR btl.openib.1.9
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 31069831169
[cn1:14224] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 31069831169
[cn1:14224] [[15336,1],0] store:peer:modex: completed modex entry for proc [[15336,1],7]
[cn1:14224] [[15336,1],0] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],0]
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],0] FOR MPI_THREAD_LEVEL
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],0] FOR btl.openib.1.9
[cn1:14224] [[15336,1],0] store:peer:modex: completed modex entry for proc [[15336,1],0]
[cn1:14224] [[15336,1],0] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],1]
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],1] FOR MPI_THREAD_LEVEL
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 5300027393
[cn1:14224] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 5300027393
[cn1:14224] [[15336,1],0] STORING MODEX DATA FROM [[15336,1],1] FOR btl.openib.1.9
[cn1:14224] storing data in NONPEER dstore
[cn1:14224] dstore:hash:store storing data for proc 5300027393
[cn1:14224] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 5300027393
[cn1:14224] [[15336,1],0] store:peer:modex: completed modex entry for proc [[15336,1],1]
[cn1:14224] [[15336,1],0] store:peer:modex NO MODEX RELEASE CBFUNC
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 1330
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 1330
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 1330
[cn1:14225] [[15336,1],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14225] [[15336,1],1]:tcp:recv:handler CONNECTED
[cn1:14225] [[15336,1],1]:tcp:recv:handler allocate new recv msg
[cn1:14225] [[15336,1],1]:tcp:recv:handler read hdr
[cn1:14225] [[15336,1],1]:tcp:recv:handler allocate data region of size 1300
[cn1:14225] [[15336,1],1] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 1300 BYTES FOR DEST [[15336,1],1] TAG 30
[cn1:14225] [[15336,1],1] DELIVERING TO RML
[cn1:14225] [[15336,1],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14225] [[15336,1],1] message received 1300 bytes from [[15336,0],0] for tag 30
[cn1:14225] [[15336,1],1] grpcomm:base:receive processing collective return for id 0 recvd from [[15336,0],0]
[cn1:14225] [[15336,1],1] CHECKING COLL id 0
[cn1:14225] [[15336,1],1] STORING MODEX DATA
[cn1:14225] [[15336,1],1] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],8]
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],8] FOR MPI_THREAD_LEVEL
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 35364798465
[cn1:14225] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 35364798465
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],8] FOR btl.openib.1.9
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 35364798465
[cn1:14225] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 35364798465
[cn1:14225] [[15336,1],1] store:peer:modex: completed modex entry for proc [[15336,1],8]
[cn1:14225] [[15336,1],1] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],9]
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],9] FOR MPI_THREAD_LEVEL
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 39659765761
[cn1:14225] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 39659765761
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],9] FOR btl.openib.1.9
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 39659765761
[cn1:14225] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 39659765761
[cn1:14225] [[15336,1],1] store:peer:modex: completed modex entry for proc [[15336,1],9]
[cn1:14225] [[15336,1],1] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],4]
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],4] FOR MPI_THREAD_LEVEL
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 18184929281
[cn1:14225] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 18184929281
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],4] FOR btl.openib.1.9
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 18184929281
[cn1:14225] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 18184929281
[cn1:14225] [[15336,1],1] store:peer:modex: completed modex entry for proc [[15336,1],4]
[cn1:14225] [[15336,1],1] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],5]
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],5] FOR MPI_THREAD_LEVEL
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 22479896577
[cn1:14225] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 22479896577
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],5] FOR btl.openib.1.9
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 22479896577
[cn1:14225] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 22479896577
[cn1:14225] [[15336,1],1] store:peer:modex: completed modex entry for proc [[15336,1],5]
[cn1:14225] [[15336,1],1] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],2]
[cn1:1[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key opal.locality on proc 5300027393
[cn1:14224] dstore_hash:fetch key opal.locality for proc 5300027393 not found
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 5300027393
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key opal.cpuset on proc 5300027393
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 5300027393
[cn1:14224] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 5300027393
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],1] for orte.hostname
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 5300027393
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key opal.locality on proc 9594994689
[cn1:14224] dstore_hash:fetch key opal.locality for proc 9594994689 not found
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 9594994689
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostid on proc 9594994689
[cn1:14224] dstore_hash:fetch key orte.hostid for proc 9594994689 not found
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 9594994689
[cn1:14224] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 9594994689
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],2] for orte.hostname
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 9594994689
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key opal.locality on proc 13889961985
[cn1:14224] dstore_hash:fetch key opal.locality for proc 13889961985 not found
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 13889961985
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostid on proc 13889961985
[cn1:14224] dstore_hash:fetch key orte.hostid for proc 13889961985 not found
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 13889961985
[cn1:14224] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 13889961985
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],3] for orte.hostname
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 13889961985
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key opal.locality on proc 18184929281
[cn1:14224] dstore_hash:fetch key opal.locality for proc 18184929281 not found
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 18184929281
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostid on proc 18184929281
[cn1:14224] dstore_hash:fetch key orte.hostid for proc 18184929281 not found
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 18184929281
[cn1:14224] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 18184929281
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],4] for orte.hostname
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 18184929281
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key opal.locality on proc 22479896577
[cn1:14224] dstore_hash:fetch key opal.locality for proc 22479896577 not found
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 22479896577
4225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],2] FOR MPI_THREAD_LEVEL
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 9594994689
[cn1:14225] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 9594994689
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],2] FOR btl.openib.1.9
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 9594994689
[cn1:14225] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 9594994689
[cn1:14225] [[15336,1],1] store:peer:modex: completed modex entry for proc [[15336,1],2]
[cn1:14225] [[15336,1],1] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],3]
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],3] FOR MPI_THREAD_LEVEL
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 13889961985
[cn1:14225] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 13889961985
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],3] FOR btl.openib.1.9
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 13889961985
[cn1:14225] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 13889961985
[cn1:14225] [[15336,1],1] store:peer:modex: completed modex entry for proc [[15336,1],3]
[cn1:14225] [[15336,1],1] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],6]
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],6] FOR MPI_THREAD_LEVEL
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 26774863873
[cn1:14225] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 26774863873
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],6] FOR btl.openib.1.9
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 26774863873
[cn1:14225] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 26774863873
[cn1:14225] [[15336,1],1] store:peer:modex: completed modex entry for proc [[15336,1],6]
[cn1:14225] [[15336,1],1] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],7]
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],7] FOR MPI_THREAD_LEVEL
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 31069831169
[cn1:14225] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 31069831169
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],7] FOR btl.openib.1.9
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 31069831169
[cn1:14225] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 31069831169
[cn1:14225] [[15336,1],1] store:peer:modex: completed modex entry for proc [[15336,1],7]
[cn1:14225] [[15336,1],1] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],0]
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],0] FOR MPI_THREAD_LEVEL
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 1005060097
[cn1:14225] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 1005060097
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],0] FOR btl.openib.1.9
[cn1:14225] storing data in NONPEER dstore
[cn1:14225] dstore:hash:store storing data for proc 1005060097
[cn1:14225] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 1005060097
[cn1:14225] [[15336,1],1] store:peer:modex: completed modex entry for proc [[15336,1],0]
[cn1:14225] [[15336,1],1] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],1]
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],1] FOR MPI_THREAD_LEVEL
[cn1:14225] [[15336,1],1] STORING MODEX DATA FROM [[15336,1],1] FOR btl.openib.1.9
[cn1:14225] [[15336,1],1] store:peer:modex: completed modex entry for proc [[15336,1],1]
[cn1:14225] [[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostid on proc 22479896577
[cn1:14224] dstore_hash:fetch key orte.hostid for proc 22479896577 not found
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 22479896577
[cn1:14224] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 22479896577
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],5] for orte.hostname
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 22479896577
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key opal.locality on proc 26774863873
[cn1:14224] dstore_hash:fetch key opal.locality for proc 26774863873 not found
[15336,1],1] store:peer:modex NO MODEX RELEASE CBFUNC
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 26774863873
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostid on proc 26774863873
[cn1:14224] dstore_hash:fetch key orte.hostid for proc 26774863873 not found
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 26774863873
[cn1:14224] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 26774863873
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],6] for orte.hostname
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 26774863873
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key opal.locality on proc 31069831169
[cn1:14224] dstore_hash:fetch key opal.locality for proc 31069831169 not found
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 31069831169
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostid on proc 31069831169
[cn1:14224] dstore_hash:fetch key orte.hostid for proc 31069831169 not found
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 31069831169
[cn1:14224] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 31069831169
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],7] for orte.hostname
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 31069831169
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key opal.locality on proc 35364798465
[cn1:14224] dstore_hash:fetch key opal.locality for proc 35364798465 not found
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 35364798465
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostid on proc 35364798465
[cn1:14224] dstore_hash:fetch key orte.hostid for proc 35364798465 not found
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 35364798465
[cn1:14224] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 35364798465
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],8] for orte.hostname
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 35364798465
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key opal.locality on proc 39659765761
[cn1:14224] dstore_hash:fetch key opal.locality for proc 39659765761 not found
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 39659765761
[cn1:14224] fetching data from INTERNAL dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostid on proc 39659765761
[cn1:14224] dstore_hash:fetch key orte.hostid for proc 39659765761 not found
[cn1:14224] storing data in INTERNAL dstore
[cn1:14224] dstore:hash:store storing data for proc 39659765761
[cn1:14224] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 39659765761
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],9] for orte.hostname
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key orte.hostname on proc 39659765761
[cn1:14224] check:select: modex not reqd
[cn1:14224] mca: bml: Using self btl to [[15336,1],0] on node cn1
[cn1:14225] fetching data from INTERNAL dstore
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:175:btl_openib_async_commandh] [cn1:14225] dstore:hash:fetch: searching for key opal.locality on proc 1005060097
[cn1:14225] dstore_hash:fetch key opal.locality for proc 1005060097 not found
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 1005060097
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key opal.cpuset on proc 1005060097
Got cmd 0[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 1005060097
[cn1:14225] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 1005060097
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],0] for orte.hostname
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 1005060097
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key opal.locality on proc 9594994689
[cn1:14225] dstore_hash:fetch key opal.locality for proc 9594994689 not found
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 9594994689
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostid on proc 9594994689
[cn1:14225] dstore_hash:fetch key orte.hostid for proc 9594994689 not found
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 9594994689
[cn1:14225] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 9594994689
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],2] for orte.hostname
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 9594994689
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key opal.locality on proc 13889961985
[cn1:14225] dstore_hash:fetch key opal.locality for proc 13889961985 not found
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 13889961985
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostid on proc 13889961985
[cn1:14225] dstore_hash:fetch key orte.hostid for proc 13889961985 not found
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 13889961985
[cn1:14225] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 13889961985
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],3] for orte.hostname
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 13889961985
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key opal.locality on proc 18184929281
[cn1:14225] dstore_hash:fetch key opal.locality for proc 18184929281 not found
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 18184929281
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostid on proc 18184929281
[cn1:14225] dstore_hash:fetch key orte.hostid for proc 18184929281 not found
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 18184929281
[cn1:14225] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 18184929281
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],4] for orte.hostname
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 18184929281
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key opal.locality on proc 22479896577
[cn1:14225] dstore_hash:fetch key opal.locality for proc 22479896577 not found
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 22479896577
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostid on proc 22479896577
[cn1:14225] dstore_hash:fetch key orte.hostid for proc 22479896577 not found
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 22479896577
[cn1:14225] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 22479896577
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],5] for orte.hostname
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 22479896577
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key opal.locality on proc 26774863873
[cn1:14225] dstore_hash:fetch key opal.locality for proc 26774863873 not found
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 26774863873
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostid on proc 26774863873
[cn1:14225] dstore_hash:fetch key orte.hostid for proc 26774863873 not found
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 26774863873
[cn1:14225] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 26774863873
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],6] for orte.hostname
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 26774863873
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key opal.locality on proc 31069831169
[cn1:14225] dstore_hash:fetch key opal.locality for proc 31069831169 not found
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 31069831169
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostid on proc 31069831169
[cn1:14225] dstore_hash:fetch key orte.hostid for proc 31069831169 not found
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 31069831169
[cn1:14225] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 31069831169
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],7] for orte.hostname
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 31069831169
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key opal.locality on proc 35364798465
[cn1:14225] dstore_hash:fetch key opal.locality for proc 35364798465 not found
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 35364798465
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostid on proc 35364798465
[cn1:14225] dstore_hash:fetch key orte.hostid for proc 35364798465 not found
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 35364798465
[cn1:14225] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 35364798465
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],8] for orte.hostname
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 35364798465
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key opal.locality on proc 39659765761
[cn1:14225] dstore_hash:fetch key opal.locality for proc 39659765761 not found
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 39659765761
[cn1:14225] fetching data from INTERNAL dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostid on proc 39659765761
[cn1:14225] dstore_hash:fetch key orte.hostid for proc 39659765761 not found
[cn1:14225] storing data in INTERNAL dstore
[cn1:14225] dstore:hash:store storing data for proc 39659765761
[cn1:14225] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 39659765761
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],9] for orte.hostname
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key orte.hostname on proc 39659765761

[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:178:btl_openib_async_commandh] Got fd 20[cn1:14225] check:select: modex not reqd
[cn1:14225] mca: bml: Using self btl to [[15336,1],1] on node cn1

[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:180:btl_openib_async_commandh] [cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:175:btl_openib_async_commandh] Got cmd 0
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:178:btl_openib_async_commandh] Got fd 20
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:180:btl_openib_async_commandh] Adding device [20] to async event poll[1]
Adding device [20] to async event poll[1]
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37181 10003 (offset now 38)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37962 10004 (offset now 38)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/b[cn1:14224] [[15336,1],0] fetch data from [[15336,1],1] for btl.openib.1.9
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 5300027393
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],2] for btl.openib.1.9
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 9594994689
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],3] for btl.openib.1.9
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 13889961985
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],4] for btl.openib.1.9
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 18184929281
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],5] for btl.openib.1.9
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 22479896577
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],6] for btl.openib.1.9
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 26774863873
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],7] for btl.openib.1.9
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 31069831169
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],8] for btl.openib.1.9
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 35364798465
[cn1:14224] [[15336,1],0] fetch data from [[15336,1],9] for btl.openib.1.9
[cn1:14224] fetching data from NONPEER dstore
[cn1:14224] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 39659765761
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],0] for btl.openib.1.9
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 1005060097
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],2] for btl.openib.1.9
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 9594994689
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],3] for btl.openib.1.9
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 13889961985
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],4] for btl.openib.1.9
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 18184929281
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],5] for btl.openib.1.9
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 22479896577
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],6] for btl.openib.1.9
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 26774863873
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],7] for btl.openib.1.9
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 31069831169
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],8] for btl.openib.1.9
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 35364798465
[cn1:14225] [[15336,1],1] fetch data from [[15336,1],9] for btl.openib.1.9
[cn1:14225] fetching data from NONPEER dstore
[cn1:14225] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 39659765761
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37182 10003 (offset now 38)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37962 10004 (offset now 38)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37963 10004 (offset now 38)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33170 10007 (offset now 38)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked bttl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37963 10004 (offset now 38)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33170 10007 (offset now 38)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33171 10007 (offset now 38)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32327 10005 (offset now 38)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[153l 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33171 10007 (offset now 38)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32327 10005 (offset now 38)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32328 10005 (offset now 38)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27543 10008 (offset now 38)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca36,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32328 10005 (offset now 38)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27543 10008 (offset now 38)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27544 10008 (offset now 38)
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27544 10008 (offset now 38)
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn1:14224] mca: bml: Using openib btl to [[15336,1],1] on node cn1
[cn1:14224] mca: bml: Using openib btl to [[15336,1],2] on node cn2
[cn1:14224] mca: bml: Using openib btl to [[15336,1],3] on node cn2
[cn1:14224] mca: bml: Using openib btl to [[15336,1],4] on node cn3
[cn1:14224] mca: bml: Using openib btl to [[15336,1],5] on node cn3
[cn1:14224] mca: bml: Using openib btl to [[15336,1],6] on node cn4
[cn1:14224] mca: bml: Using openib btl to [[15336,1],7] on node cn4
[cn1:14224] mca: bml: Using openib btl to [[15336,1],8] on node cn5
[cn1:14224] mca: bml: Using openib btl to [[15336,1],9] on node cn5
[cn1:14224] [[15336,1],0] grpcomm:bad entering barrier
[cn1:14224] [[15336,1],0] ACTIVATING GRCPCOMM OP 1 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 10
[cn1:14225] mca: bml: Using openib btl to [[15336,1],0] on node cn1
[cn1:14225] mca: bml: Using openib btl to [[15336,1],2] on node cn2
[cn1:14225] mca: bml: Using openib btl to [[15336,1],3] on node cn2
[cn1:14225] mca: bml: Using openib btl to [[15336,1],4] on node cn3
[cn1:14225] mca: bml: Using openib btl to [[15336,1],5] on node cn3
[cn1:14225] mca: bml: Using openib btl to [[15336,1],6] on node cn4
[cn1:14225] mca: bml: Using openib btl to [[15336,1],7] on node cn4
[cn1:14225] mca: bml: Using openib btl to [[15336,1],8] on node cn5
[cn1:14225] mca: bml: Using openib btl to [[15336,1],9] on node cn5
[cn1:14225] [[15336,1],1] grpcomm:bad entering barrier
[cn1:14225] [[15336,1],1] ACTIVATING GRCPCOMM OP 1 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn1:14225] [[15336,1],1] rml_send_buffer to peer [[15336,0],0] at tag 30
[cn1:14225] [[15336,1],1] grpcomm:bad barrier underway
[cn1:14225] [[15336,1],1] rml_send_msg to peer [[15336,0],0] at tag 30
[cn1:14225] [[15336,1],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14225] [[15336,1],1] oob:base:send to target [[15336,0],0]
[cn1:14225] [[15336,1],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14225] [[15336,1],1] oob:tcp:send_nb to peer [[15336,0],0]:30
[cn1:14225] [[15336,1],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:30
[cn1:14225] [[15336,1],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14225] [[15336,1],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 10 BYTES ON SOCKET 11
[cn1:14225] [[15336,1],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14224] [[15336,1],0] rml_send_buffer to peer [[15336,0],0] at tag 30
[cn1:14224] [[15336,1],0] grpcomm:bad barrier underway
[cn1:14224] [[15336,1],0] rml_send_msg to peer [[15336,0],0] at tag 30
[cn1:14224] [[15336,1],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14224] [[15336,1],0] oob:base:send to target [[15336,0],0]
[cn1:14224] [[15336,1],0] oob:base:send known transport for peer [[15336,0],0]
[cn1:14224] [[15336,1],0] oob:tcp:send_nb to peer [[15336,0],0]:30
[cn1:14224] [[15336,1],0] tcp:send_nb to peer [[15336,0],0]
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:30
[cn1:14224] [[15336,1],0] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 10
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14224] [[15336,1],0] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 10 BYTES ON SOCKET 11
[cn1:14224] [[15336,1],0]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 1330 BYTES FOR DEST [[15336,0],1] TAG 15
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] message received 1330 bytes from [[15336,0],0] for tag 15
[cn2:01042] [[15336,0],1] grpcomm:xcast:recv: with 1330 bytes
[cn2:01042] [[15336,0],1] orte:daemon:send_relay sending relay msg to [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],3] at tag 15
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],1] at tag 1
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],3] at tag 15
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],1] at tag 1
[cn2:01042] [[15336,0],1] rml_send_iovec_to_self at tag 1
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],3]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],3]:15
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_to_self callback executing for tag 1
[cn2:01042] [[15336,0],1] message received 1330 bytes from [[15336,0],1] for tag 1
[cn2:01042] [[15336,0],1] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn2:01042] [[15336,0],1] orted_cmd: received message_local_procs
[cn2:01042] [[15336,0],1] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,1],2] at tag 30
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,1],3] at tag 30
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:15
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],3] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,1],2] at tag 30
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,1],3] at tag 30
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,1],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,1],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,1],2]:30
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,1],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,1],3]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,1],3]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,1],3]:30
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,1],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1176
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],2]:30
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,1],2]) --> [[15336,1],2]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,1],2] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],3]:30
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,1],3]) --> [[15336,1],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,1],3] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],3]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,1],3]
[cn1:14209] [[15336,0],0] message received 1176 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 1330
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,1],3]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,1],3] OF 1300 BYTES ON SOCKET 14
[cn2:01046] [[15336,1],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01046] [[15336,1],3]:tcp:recv:handler CONNECTED
[cn2:01046] [[15336,1],3]:tcp:recv:handler allocate new recv msg
[cn2:01046] [[15336,1],3]:tcp:recv:handler read hdr
[cn2:01046] [[15336,1],3]:tcp:recv:handler allocate data region of size 1300
[cn2:01046] [[15336,1],3] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1300 BYTES FOR DEST [[15336,1],3] TAG 30
[cn2:01046] [[15336,1],3] DELIVERING TO RML
[cn2:01046] [[15336,1],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] message received 1300 bytes from [[15336,0],1] for tag 30
[cn2:01046] [[15336,1],3] grpcomm:base:receive processing collective return for id 0 recvd from [[15336,0],1]
[cn2:01046] [[15336,1],3] CHECKING COLL id 0
[cn2:01046] [[15336,1],3] STORING MODEX DATA
[cn2:01046] [[15336,1],3] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],8]
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],8] FOR MPI_THREAD_LEVEL
[cn2:01046] storing data in NONPEER dstore
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]-[[15336,1],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1337
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 1330 BYTES ON SOCKET 10
[cn2:01042] [[15336,0],1]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1337 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,1],2]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,1],2]
[cn1:14209] [[15336,0],0] message received 1337 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,1],2] OF 1300 BYTES ON SOCKET 13
[cn2:01042] [[15336,0],1]-[[15336,1],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2241
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01046] dstore:hash:store storing data for proc 35364798465
[cn2:01046] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 35364798465
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],8] FOR btl.openib.1.9
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 35364798465
[cn2:01046] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 35364798465
[cn2:01046] [[15336,1],3] store:peer:modex: completed modex entry for proc [[15336,1],8]
[cn2:01046] [[15336,1],3] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],9]
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],9] FOR MPI_THREAD_LEVEL
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 39659765761
[cn2:01046] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 39659765761
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],9] FOR btl.openib.1.9
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 39659765761
[cn2:01046] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 39659765761
[cn2:01046] [[15336,1],3] store:peer:modex: completed modex entry for proc [[15336,1],9]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1330 BYTES FOR DEST [[15336,0],3] TAG 15
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] message received 1330 bytes from [[15336,0],1] for tag 15
[cn4:17828] [[15336,0],3] grpcomm:xcast:recv: with 1330 bytes
[cn4:17828] [[15336,0],3] orte:daemon:send_relay - recipient list is empty!
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],3] at tag 1
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],3] at tag 1
[cn4:17828] [[15336,0],3] rml_send_iovec_to_self at tag 1
[cn4:17828] [[15336,0],3] rml_send_to_self callback executing for tag 1
[cn4:17828] [[15336,0],3] message received 1330 bytes from [[15336,0],3] for tag 1
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1337 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 1330 BYTES FOR DEST [[15336,0],4] TAG 15
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] message received 1330 bytes from [[15336,0],0] for tag 15
[cn5:14171] [[15336,0],4] grpcomm:xcast:recv: with 1330 bytes
[cn5:14171] [[15336,0],4] orte:daemon:send_relay - recipient list is empty!
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],4] at tag 1
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],4] at tag 1
[cn5:14171] [[15336,0],4] rml_send_iovec_to_self at tag 1
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2241 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 2241 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn4:17828] [[15336,0],3] orted_cmd: received message_local_procs
[cn4:17828] [[15336,0],3] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn2:01046] [[15336,1],3] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],4]
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],4] FOR MPI_THREAD_LEVEL
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 18184929281
[cn2:01046] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 18184929281
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],4] FOR btl.openib.1.9
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 18184929281
[cn2:01046] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 18184929281
[cn2:01046] [[15336,1],3] store:peer:modex: completed modex entry for proc [[15336,1],4]
[cn2:01046] [[15336,1],3] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],5]
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],5] FOR MPI_THREAD_LEVEL
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 22479896577
[cn2:01046] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 22479896577
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],5] FOR btl.openib.1.9
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 22479896577
[cn2:01046] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 22479896577
[cn2:01046] [[15336,1],3] store:peer:modex: completed modex entry for proc [[15336,1],5]
[cn2:01046] [[15336,1],3] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],2]
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],2] FOR MPI_THREAD_LEVEL
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 9594994689
[cn2:01046] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 9594994689
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],2] FOR btl.openib.1.9
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 9594994689
[cn2:01046] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 9594994689
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_to_self callback executing for tag 1
[cn5:14171] [[15336,0],4] message received 1330 bytes from [[15336,0],4] for tag 1
[cn5:14171] [[15336,0],4] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn5:14171] [[15336,0],4] orted_cmd: received message_local_procs
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2013
[cn5:14171] [[15336,0],4] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2013 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,1],6] at tag 30
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,1],7] at tag 30
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,1],6] at tag 30
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,1],7] at tag 30
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 2013 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,1],8] at tag 30
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,1],9] at tag 30
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,1],8] at tag 30
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,1],9] at tag 30
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,1],6]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,1],6]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,1],6]:30
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,1],6]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],6]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,1],7]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,1],7]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1625
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01046] [[15336,1],3] store:peer:modex: completed modex entry for proc [[15336,1],2]
[cn2:01046] [[15336,1],3] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],3]
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],3] FOR MPI_THREAD_LEVEL
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],3] FOR btl.openib.1.9
[cn2:01046] [[15336,1],3] store:peer:modex: completed modex entry for proc [[15336,1],3]
[cn2:01046] [[15336,1],3] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],6]
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],6] FOR MPI_THREAD_LEVEL
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 26774863873
[cn2:01046] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 26774863873
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],6] FOR btl.openib.1.9
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 26774863873
[cn2:01046] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 26774863873
[cn2:01046] [[15336,1],3] store:peer:modex: completed modex entry for proc [[15336,1],6]
[cn2:01046] [[15336,1],3] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],7]
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],7] FOR MPI_THREAD_LEVEL
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 31069831169
[cn2:01046] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 31069831169
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],7] FOR btl.openib.1.9
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 31069831169
[cn2:01046] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 31069831169
[cn2:01046] [[15336,1],3] store:peer:modex: completed modex entry for proc [[15336,1],7]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,1],8]:30
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,1],7]:30
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,1],7]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],7]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],6]:30
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 947
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2241 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 947 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,1],6]) --> [[15336,1],6]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,1],6] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],6]
[cn1:14209] [[15336,0],0] message received 947 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],7]:30
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,1],7]) --> [[15336,1],7]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1625 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,1],7] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],7]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1027
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,1],6]
[cn4:17831] [[15336,1],6]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17831] [[15336,1],6]:tcp:recv:handler CONNECTED
[cn4:17831] [[15336,1],6]:tcp:recv:handler allocate new recv msg
[cn4:17831] [[15336,1],6]:tcp:recv:handler read hdr
[cn4:17831] [[15336,1],6]:tcp:recv:handler allocate data region of size 1300
[cn4:17831] [[15336,1],6] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1300 BYTES FOR DEST [[15336,1],6] TAG 30
[cn4:17831] [[15336,1],6] DELIVERING TO RML
[cn4:17831] [[15336,1],6] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] [[15336,1],6] message received 1300 bytes from [[15336,0],3] for tag 30
[cn4:17831] [[15336,1],6] grpcomm:base:receive processing collective return for id 0 recvd from [[15336,0],3]
[cn4:17831] [[15336,1],6] CHECKING COLL id 0
[cn4:17831] [[15336,1],6] STORING MODEX DATA
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,1],6]
[cn1:14209] [[15336,0],0] message received 1625 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,1],6] OF 1300 BYTES ON SOCKET 13
[cn4:17828] [[15336,0],3]-[[15336,1],6] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01046] [[15336,1],3] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],0]
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],0] FOR MPI_THREAD_LEVEL
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 1005060097
[cn2:01046] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 1005060097
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],0] FOR btl.openib.1.9
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 1005060097
[cn2:01046] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 1005060097
[cn2:01046] [[15336,1],3] store:peer:modex: completed modex entry for proc [[15336,1],0]
[cn2:01046] [[15336,1],3] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],1]
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],1] FOR MPI_THREAD_LEVEL
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 5300027393
[cn2:01046] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 5300027393
[cn2:01046] [[15336,1],3] STORING MODEX DATA FROM [[15336,1],1] FOR btl.openib.1.9
[cn2:01046] storing data in NONPEER dstore
[cn2:01046] dstore:hash:store storing data for proc 5300027393
[cn2:01046] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 5300027393
[cn2:01046] [[15336,1],3] store:peer:modex: completed modex entry for proc [[15336,1],1]
[cn2:01046] [[15336,1],3] store:peer:modex NO MODEX RELEASE CBFUNC
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1027 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,1],7]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,1],7]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1027 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,1],7] OF 1300 BYTES ON SOCKET 14
[cn4:17831] [[15336,1],6] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],8]
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],8] FOR MPI_THREAD_LEVEL
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 35364798465
[cn4:17831] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 35364798465
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],8] FOR btl.openib.1.9
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 35364798465
[cn4:17831] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 35364798465
[cn4:17831] [[15336,1],6] store:peer:modex: completed modex entry for proc [[15336,1],8]
[cn4:17831] [[15336,1],6] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],9]
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],9] FOR MPI_THREAD_LEVEL
[cn4:17831] storing data in NONPEER dstore
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]-[[15336,1],7] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1729
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 1330 BYTES FOR DEST [[15336,0],2] TAG 15
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] message received 1330 bytes from [[15336,0],0] for tag 15
[cn3:21416] [[15336,0],2] grpcomm:xcast:recv: with 1330 bytes
[cn3:21416] [[15336,0],2] orte:daemon:send_relay - recipient list is empty!
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],2] at tag 1
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],2] at tag 1
[cn3:21416] [[15336,0],2] rml_send_iovec_to_self at tag 1
[cn3:21416] [[15336,0],2] rml_send_to_self callback executing for tag 1
[cn3:21416] [[15336,0],2] message received 1330 bytes from [[15336,0],2] for tag 1
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2013 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,1],8]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,1],9]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,1],9]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,1],9]:30
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,1],9]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],9]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],8]:30
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,1],8]) --> [[15336,1],8]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,1],8] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],8]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],9]:30
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,1],9]) --> [[15336,1],9]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,1],9] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],9]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1729 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1729 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,1],8]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,1],8]
[cn4:17831] dstore:hash:store storing data for proc 39659765761
[cn4:17831] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 39659765761
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],9] FOR btl.openib.1.9
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 39659765761
[cn4:17831] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 39659765761
[cn4:17831] [[15336,1],6] store:peer:modex: completed modex entry for proc [[15336,1],9]
[cn4:17831] [[15336,1],6] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],4]
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],4] FOR MPI_THREAD_LEVEL
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 18184929281
[cn4:17831] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 18184929281
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],4] FOR btl.openib.1.9
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 18184929281
[cn4:17831] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 18184929281
[cn4:17831] [[15336,1],6] store:peer:modex: completed modex entry for proc [[15336,1],4]
[cn4:17831] [[15336,1],6] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],5]
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],5] FOR MPI_THREAD_LEVEL
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 22479896577
[cn4:17831] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 22479896577
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1625 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,1],8] OF 1300 BYTES ON SOCKET 12
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,1],8] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 350
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 350 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,1],9]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,1],9]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1645
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,1],9] OF 1300 BYTES ON SOCKET 13
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1645 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,1],9] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 902
[cn3:21416] [[15336,0],2] orted_cmd: received message_local_procs
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 902 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,1],4] at tag 30
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,1],5] at tag 30
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,1],4] at tag 30
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 350 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 1645 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,1],5] at tag 30
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,1],4]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,1],4]
[cn1:14209] [[15336,0],0] message received 902 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,1],4]:30
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,1],4]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],4]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,1],5]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,1],5]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2124
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,1],5]:30
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,1],5]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],5]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],4]:30
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 2124 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,1],4]) --> [[15336,1],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,1],4] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1522
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1522 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],5]:30
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,1],5]) --> [[15336,1],5]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,1],5] - queueing for send
[cn5:14175] [[15336,1],9]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14175] [[15336,1],9]:tcp:recv:handler CONNECTED
[cn5:14175] [[15336,1],9]:tcp:recv:handler allocate new recv msg
[cn5:14175] [[15336,1],9]:tcp:recv:handler read hdr
[cn5:14175] [[15336,1],9]:tcp:recv:handler allocate data region of size 1300
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],5] FOR btl.openib.1.9
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 22479896577
[cn4:17831] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 22479896577
[cn4:17831] [[15336,1],6] store:peer:modex: completed modex entry for proc [[15336,1],5]
[cn4:17831] [[15336,1],6] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],2]
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],2] FOR MPI_THREAD_LEVEL
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 9594994689
[cn4:17831] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 9594994689
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],2] FOR btl.openib.1.9
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 9594994689
[cn4:17831] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 9594994689
[cn4:17831] [[15336,1],6] store:peer:modex: completed modex entry for proc [[15336,1],2]
[cn4:17831] [[15336,1],6] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],3]
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],3] FOR MPI_THREAD_LEVEL
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 13889961985
[cn4:17831] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 13889961985
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],3] FOR btl.openib.1.9
[cn3:21419] [[15336,1],4]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21419] [[15336,1],4]:tcp:recv:handler CONNECTED
[cn3:21419] [[15336,1],4]:tcp:recv:handler allocate new recv msg
[cn3:21419] [[15336,1],4]:tcp:recv:handler read hdr
[cn3:21419] [[15336,1],4]:tcp:recv:handler allocate data region of size 1300
[cn3:21419] [[15336,1],4] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1300 BYTES FOR DEST [[15336,1],4] TAG 30
[cn3:21419] [[15336,1],4] DELIVERING TO RML
[cn3:21419] [[15336,1],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] [[15336,1],4] message received 1300 bytes from [[15336,0],2] for tag 30
[cn3:21419] [[15336,1],4] grpcomm:base:receive processing collective return for id 0 recvd from [[15336,0],2]
[cn3:21419] [[15336,1],4] CHECKING COLL id 0
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],5]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,1],4]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,1],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 920
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 920 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 2124 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 947 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 1522 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 920 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,1],4] OF 1300 BYTES ON SOCKET 12
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]-[[15336,1],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1027 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,1],5]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,1],5]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 350 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1631
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1631 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,1],5] OF 1300 BYTES ON SOCKET 13
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]-[[15336,1],5] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 950
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 950 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1156
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1156 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 13889961985
[cn4:17831] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 13889961985
[cn4:17831] [[15336,1],6] store:peer:modex: completed modex entry for proc [[15336,1],3]
[cn4:17831] [[15336,1],6] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],6]
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],6] FOR MPI_THREAD_LEVEL
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],6] FOR btl.openib.1.9
[cn4:17831] [[15336,1],6] store:peer:modex: completed modex entry for proc [[15336,1],6]
[cn4:17831] [[15336,1],6] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],7]
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],7] FOR MPI_THREAD_LEVEL
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 31069831169
[cn4:17831] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 31069831169
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],7] FOR btl.openib.1.9
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 31069831169
[cn4:17831] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 31069831169
[cn4:17831] [[15336,1],6] store:peer:modex: completed modex entry for proc [[15336,1],7]
[cn4:17831] [[15336,1],6] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],0]
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],0] FOR MPI_THREAD_LEVEL
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 1005060097
[cn4:17831] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 1005060097
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],0] FOR btl.openib.1.9
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 1005060097
[cn4:17831] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 1005060097
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21419] [[15336,1],4] STORING MODEX DATA
[cn3:21419] [[15336,1],4] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],8]
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],8] FOR MPI_THREAD_LEVEL
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 35364798465
[cn3:21419] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 35364798465
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],8] FOR btl.openib.1.9
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 35364798465
[cn3:21419] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 35364798465
[cn3:21419] [[15336,1],4] store:peer:modex: completed modex entry for proc [[15336,1],8]
[cn3:21419] [[15336,1],4] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],9]
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],9] FOR MPI_THREAD_LEVEL
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 39659765761
[cn3:21419] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 39659765761
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],9] FOR btl.openib.1.9
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 39659765761
[cn3:21419] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 39659765761
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14175] [[15336,1],9] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1300 BYTES FOR DEST [[15336,1],9] TAG 30
[cn5:14175] [[15336,1],9] DELIVERING TO RML
[cn5:14175] [[15336,1],9] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14175] [[15336,1],9] message received 1300 bytes from [[15336,0],4] for tag 30
[cn5:14175] [[15336,1],9] grpcomm:base:receive processing collective return for id 0 recvd from [[15336,0],4]
[cn5:14175] [[15336,1],9] CHECKING COLL id 0
[cn5:14175] [[15336,1],9] STORING MODEX DATA
[cn5:14175] [[15336,1],9] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],8]
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],8] FOR MPI_THREAD_LEVEL
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 35364798465
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 1631 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 950 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1156 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14175] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 35364798465
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],8] FOR btl.openib.1.9
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 35364798465
[cn5:14175] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 35364798465
[cn5:14175] [[15336,1],9] store:peer:modex: completed modex entry for proc [[15336,1],8]
[cn5:14175] [[15336,1],9] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],9]
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],9] FOR MPI_THREAD_LEVEL
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],9] FOR btl.openib.1.9
[cn5:14175] [[15336,1],9] store:peer:modex: completed modex entry for proc [[15336,1],9]
[cn5:14175] [[15336,1],9] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],4]
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],4] FOR MPI_THREAD_LEVEL
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 18184929281
[cn5:14175] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 18184929281
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],4] FOR btl.openib.1.9
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 18184929281
[cn5:14175] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 18184929281
[cn5:14175] [[15336,1],9] store:peer:modex: completed modex entry for proc [[15336,1],4]
[cn4:17831] [[15336,1],6] store:peer:modex: completed modex entry for proc [[15336,1],0]
[cn4:17831] [[15336,1],6] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],1]
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],1] FOR MPI_THREAD_LEVEL
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 5300027393
[cn4:17831] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 5300027393
[cn4:17831] [[15336,1],6] STORING MODEX DATA FROM [[15336,1],1] FOR btl.openib.1.9
[cn4:17831] storing data in NONPEER dstore
[cn4:17831] dstore:hash:store storing data for proc 5300027393
[cn4:17831] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 5300027393
[cn4:17831] [[15336,1],6] store:peer:modex: completed modex entry for proc [[15336,1],1]
[cn4:17831] [[15336,1],6] store:peer:modex NO MODEX RELEASE CBFUNC
[cn3:21419] [[15336,1],4] store:peer:modex: completed modex entry for proc [[15336,1],9]
[cn3:21419] [[15336,1],4] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],4]
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],4] FOR MPI_THREAD_LEVEL
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],4] FOR btl.openib.1.9
[cn3:21419] [[15336,1],4] store:peer:modex: completed modex entry for proc [[15336,1],4]
[cn3:21419] [[15336,1],4] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],5]
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],5] FOR MPI_THREAD_LEVEL
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 22479896577
[cn3:21419] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 22479896577
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],5] FOR btl.openib.1.9
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 22479896577
[cn3:21419] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 22479896577
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1562
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1562 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1651
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1729 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1651 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 1562 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 920 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 1651 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2148
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 2148 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 902 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2039
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 2039 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14175] [[15336,1],9] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],5]
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],5] FOR MPI_THREAD_LEVEL
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 22479896577
[cn5:14175] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 22479896577
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],5] FOR btl.openib.1.9
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 22479896577
[cn5:14175] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 22479896577
[cn5:14175] [[15336,1],9] store:peer:modex: completed modex entry for proc [[15336,1],5]
[cn5:14175] [[15336,1],9] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],2]
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],2] FOR MPI_THREAD_LEVEL
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 9594994689
[cn5:14175] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 9594994689
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],2] FOR btl.openib.1.9
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 9594994689
[cn5:14175] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 9594994689
[cn5:14175] [[15336,1],9] store:peer:modex: completed modex entry for proc [[15336,1],2]
[cn3:21419] [[15336,1],4] store:peer:modex: completed modex entry for proc [[15336,1],5]
[cn3:21419] [[15336,1],4] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],2]
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],2] FOR MPI_THREAD_LEVEL
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 9594994689
[cn3:21419] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 9594994689
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],2] FOR btl.openib.1.9
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 9594994689
[cn3:21419] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 9594994689
[cn3:21419] [[15336,1],4] store:peer:modex: completed modex entry for proc [[15336,1],2]
[cn3:21419] [[15336,1],4] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],3]
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],3] FOR MPI_THREAD_LEVEL
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 13889961985
[cn3:21419] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 13889961985
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],3] FOR btl.openib.1.9
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 13889961985
[cn3:21419] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 13889961985
[cn3:21419] [[15336,1],4] store:peer:modex: completed modex entry for proc [[15336,1],3]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1645 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 2148 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 2039 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key opal.locality on proc 1005060097
[cn2:01046] dstore_hash:fetch key opal.locality for proc 1005060097 not found
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 1005060097
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostid on proc 1005060097
[cn2:01046] dstore_hash:fetch key orte.hostid for proc 1005060097 not found
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 1005060097
[cn2:01046] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 1005060097
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],0] for orte.hostname
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 1005060097
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key opal.locality on proc 5300027393
[cn2:01046] dstore_hash:fetch key opal.locality for proc 5300027393 not found
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 5300027393
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostid on proc 5300027393
[cn2:01046] dstore_hash:fetch key orte.hostid for proc 5300027393 not found
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 5300027393
[cn2:01046] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 5300027393
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],1] for orte.hostname
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 5300027393
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key opal.locality on proc 9594994689
[cn2:01046] dstore_hash:fetch key opal.locality for proc 9594994689 not found
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 9594994689
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key opal.cpuset on proc 9594994689
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 9594994689
[cn2:01046] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 9594994689
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],2] for orte.hostname
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 9594994689
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key opal.locality on proc 18184929281
[cn2:01046] dstore_hash:fetch key opal.locality for proc 18184929281 not found
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 18184929281
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostid on proc 18184929281
[cn2:01046] dstore_hash:fetch key orte.hostid for proc 18184929281 not found
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 18184929281
[cn2:01046] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 18184929281
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],4] for orte.hostname
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 18184929281
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key opal.locality on proc 22479896577
[cn2:01046] dstore_hash:fetch key opal.locality for proc 22479896577 not found
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 22479896577
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dsto[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14175] [[15336,1],9] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],3]
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],3] FOR MPI_THREAD_LEVEL
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 13889961985
[cn5:14175] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 13889961985
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],3] FOR btl.openib.1.9
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 13889961985
[cn5:14175] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 13889961985
[cn5:14175] [[15336,1],9] store:peer:modex: completed modex entry for proc [[15336,1],3]
[cn5:14175] [[15336,1],9] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],6]
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],6] FOR MPI_THREAD_LEVEL
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 26774863873
[cn5:14175] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 26774863873
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],6] FOR btl.openib.1.9
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 26774863873
[cn5:14175] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 26774863873
[cn5:14175] [[15336,1],9] store:peer:modex: completed modex entry for proc [[15336,1],6]
[cn5:14175] [[15336,1],9] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],7]
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],7] FOR MPI_THREAD_LEVEL
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 31069831169
[cn5:14175] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 31069831169
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],7] FOR btl.openib.1.9
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 31069831169
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21419] [[15336,1],4] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],6]
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],6] FOR MPI_THREAD_LEVEL
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 26774863873
[cn3:21419] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 26774863873
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],6] FOR btl.openib.1.9
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 26774863873
[cn3:21419] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 26774863873
[cn3:21419] [[15336,1],4] store:peer:modex: completed modex entry for proc [[15336,1],6]
[cn3:21419] [[15336,1],4] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],7]
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],7] FOR MPI_THREAD_LEVEL
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 31069831169
[cn3:21419] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 31069831169
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],7] FOR btl.openib.1.9
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 31069831169
[cn3:21419] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 31069831169
[cn3:21419] [[15336,1],4] store:peer:modex: completed modex entry for proc [[15336,1],7]
[cn3:21419] [[15336,1],4] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],0]
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],0] FOR MPI_THREAD_LEVEL
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 1005060097
[cn3:21419] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 1005060097
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],0] FOR btl.openib.1.9
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1631 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1655
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1655 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 1655 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01045] [[15336,1],2]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01045] [[15336,1],2]:tcp:recv:handler CONNECTED
[cn2:01045] [[15336,1],2]:tcp:recv:handler allocate new recv msg
[cn2:01045] [[15336,1],2]:tcp:recv:handler read hdr
[cn2:01045] [[15336,1],2]:tcp:recv:handler allocate data region of size 1300
[cn2:01045] [[15336,1],2] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1300 BYTES FOR DEST [[15336,1],2] TAG 30
[cn2:01045] [[15336,1],2] DELIVERING TO RML
[cn2:01045] [[15336,1],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] [[15336,1],2] message received 1300 bytes from [[15336,0],1] for tag 30
[cn2:01045] [[15336,1],2] grpcomm:base:receive processing collective return for id 0 recvd from [[15336,0],1]
[cn2:01045] [[15336,1],2] CHECKING COLL id 0
[cn2:01045] [[15336,1],2] STORING MODEX DATA
[cn2:01045] [[15336,1],2] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],8]
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],8] FOR MPI_THREAD_LEVEL
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 35364798465
[cn2:01045] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 35364798465
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],8] FOR btl.openib.1.9
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 35364798465
[cn2:01045] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 35364798465
[cn2:01045] [[15336,1],2] store:peer:modex: completed modex entry for proc [[15336,1],8]
[cn2:01045] [[15336,1],2] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],9]
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],9] FOR MPI_THREAD_LEVEL
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 39659765761
[cn2:01045] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 39659765761
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],9] FOR btl.openib.1.9
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 39659765761
[cn2:01045] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 39659765761
[cn2:01045] [[15336,1],2] store:peer:modex: completed modex entry for proc [[15336,1],9]
[cn2:01045] [[15336,1],2] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],4]
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],4] FOR MPI_THREAD_LEVEL
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 18184929281
[cn2:01045] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 18184929281
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],4] FOR btl.openib.1.9
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 18184929281
[cn2:01045] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 18184929281
[cn2:01045] [[15336,1],2] store:peer:modex: completed modex entry for proc [[15336,1],4]
[cn2:01045] [[15336,1],2] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],5]
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],5] FOR MPI_THREAD_LEVEL
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 22479896577
[cn2:01045] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 22479896577
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],5] FOR btl.openib.1.9
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 22479896577
[cn2:01045] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 22479896577
[cn2:01045] [[15336,1],2] store:peer:modex: completed modex entry for proc [[15336,1],5]
[cn2:01045] [[15336,1],2] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],2]
[cn2:0[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14175] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 31069831169
[cn5:14175] [[15336,1],9] store:peer:modex: completed modex entry for proc [[15336,1],7]
[cn5:14175] [[15336,1],9] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],0]
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],0] FOR MPI_THREAD_LEVEL
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 1005060097
[cn5:14175] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 1005060097
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],0] FOR btl.openib.1.9
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 1005060097
[cn5:14175] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 1005060097
[cn5:14175] [[15336,1],9] store:peer:modex: completed modex entry for proc [[15336,1],0]
[cn5:14175] [[15336,1],9] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],1]
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],1] FOR MPI_THREAD_LEVEL
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 5300027393
[cn5:14175] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 5300027393
[cn5:14175] [[15336,1],9] STORING MODEX DATA FROM [[15336,1],1] FOR btl.openib.1.9
[cn5:14175] storing data in NONPEER dstore
[cn5:14175] dstore:hash:store storing data for proc 5300027393
[cn5:14175] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 5300027393
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1522 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1562 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2124 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
re:hash:fetch: searching for key orte.hostid on proc 22479896577
[cn2:01046] dstore_hash:fetch key orte.hostid for proc 22479896577 not found
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 22479896577
[cn2:01046] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 22479896577
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],5] for orte.hostname
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 22479896577
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key opal.locality on proc 26774863873
[cn2:01046] dstore_hash:fetch key opal.locality for proc 26774863873 not found
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 26774863873
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostid on proc 26774863873
[cn2:01046] dstore_hash:fetch key orte.hostid for proc 26774863873 not found
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 26774863873
[cn2:01046] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 26774863873
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],6] for orte.hostname
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 26774863873
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key opal.locality on proc 31069831169
[cn2:01046] dstore_hash:fetch key opal.locality for proc 31069831169 not found
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 31069831169
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostid on proc 31069831169
[cn2:01046] dstore_hash:fetch key orte.hostid for proc 31069831169 not found
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 31069831169
[cn2:01046] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 31069831169
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],7] for orte.hostname
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 31069831169
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key opal.locality on proc 35364798465
[cn2:01046] dstore_hash:fetch key opal.locality for proc 35364798465 not found
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 35364798465
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostid on proc 35364798465
[cn2:01046] dstore_hash:fetch key orte.hostid for proc 35364798465 not found
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 35364798465
[cn2:01046] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 35364798465
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],8] for orte.hostname
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 35364798465
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key opal.locality on proc 39659765761
[cn2:01046] dstore_hash:fetch key opal.locality for proc 39659765761 not found
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 39659765761
[cn2:01046] fetching data from INTERNAL dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostid on proc 39659765761
[cn2:01046] dstore_hash:fetch key orte.hostid for proc 39659765761 not found
[cn2:01046] storing data in INTERNAL dstore
[cn2:01046] dstore:hash:store storing data for proc 39659765761
[cn2:01046] dstore:hash:store: storing key opal.locali[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14175] [[15336,1],9] store:peer:modex: completed modex entry for proc [[15336,1],1]
[cn5:14175] [[15336,1],9] store:peer:modex NO MODEX RELEASE CBFUNC
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17832] [[15336,1],7]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17832] [[15336,1],7]:tcp:recv:handler CONNECTED
[cn4:17832] [[15336,1],7]:tcp:recv:handler allocate new recv msg
[cn4:17832] [[15336,1],7]:tcp:recv:handler read hdr
[cn4:17832] [[15336,1],7]:tcp:recv:handler allocate data region of size 1300
[cn4:17832] [[15336,1],7] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1300 BYTES FOR DEST [[15336,1],7] TAG 30
[cn4:17832] [[15336,1],7] DELIVERING TO RML
[cn4:17832] [[15336,1],7] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17832] [[15336,1],7] message received 1300 bytes from [[15336,0],3] for tag 30
[cn4:17832] [[15336,1],7] grpcomm:base:receive processing collective return for id 0 recvd from [[15336,0],3]
[cn4:17832] [[15336,1],7] CHECKING COLL id 0
[cn4:17832] [[15336,1],7] STORING MODEX DATA
[cn4:17832] [[15336,1],7] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],8]
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],8] FOR MPI_THREAD_LEVEL
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 35364798465
[cn4:17832] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 35364798465
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],8] FOR btl.openib.1.9
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 35364798465
[cn4:17832] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 35364798465
[cn4:17832] [[15336,1],7] store:peer:modex: completed modex entry for proc [[15336,1],8]
[cn4:17832] [[15336,1],7] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],9]
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],9] FOR MPI_THREAD_LEVEL
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 39659765761
[cn4:17832] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 39659765761
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],9] FOR btl.openib.1.9
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 39659765761
[cn4:17832] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 39659765761
[cn4:17832] [[15336,1],7] store:peer:modex: completed modex entry for proc [[15336,1],9]
[cn4:17832] [[15336,1],7] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],4]
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],4] FOR MPI_THREAD_LEVEL
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 18184929281
[cn4:17832] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 18184929281
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],4] FOR btl.openib.1.9
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 18184929281
[cn4:17832] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 18184929281
[cn4:17832] [[15336,1],7] store:peer:modex: completed modex entry for proc [[15336,1],4]
[cn4:17832] [[15336,1],7] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],5]
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],5] FOR MPI_THREAD_LEVEL
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 22479896577
[cn4:17832] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 22479896577
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],5] FOR btl.openib.1.9
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 22479896577
[cn4:17832] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 22479896577
[cn4:17832] [[15336,1],7] store:peer:modex: completed modex entry for proc [[15336,1],5]
[cn4:17832] [[15336,1],7] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],2]
[cn4:1[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2148 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1152
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1152 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1156 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 1152 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
1045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],2] FOR MPI_THREAD_LEVEL
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],2] FOR btl.openib.1.9
[cn2:01045] [[15336,1],2] store:peer:modex: completed modex entry for proc [[15336,1],2]
[cn2:01045] [[15336,1],2] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],3]
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],3] FOR MPI_THREAD_LEVEL
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 13889961985
[cn2:01045] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 13889961985
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],3] FOR btl.openib.1.9
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 13889961985
[cn2:01045] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 13889961985
[cn2:01045] [[15336,1],2] store:peer:modex: completed modex entry for proc [[15336,1],3]
[cn2:01045] [[15336,1],2] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],6]
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],6] FOR MPI_THREAD_LEVEL
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 26774863873
[cn2:01045] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 26774863873
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],6] FOR btl.openib.1.9
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 26774863873
[cn2:01045] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 26774863873
[cn2:01045] [[15336,1],2] store:peer:modex: completed modex entry for proc [[15336,1],6]
[cn2:01045] [[15336,1],2] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],7]
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],7] FOR MPI_THREAD_LEVEL
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 31069831169
[cn2:01045] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 31069831169
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],7] FOR btl.openib.1.9
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 31069831169
[cn2:01045] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 31069831169
[cn2:01045] [[15336,1],2] store:peer:modex: completed modex entry for proc [[15336,1],7]
[cn2:01045] [[15336,1],2] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],0]
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],0] FOR MPI_THREAD_LEVEL
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 1005060097
[cn2:01045] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 1005060097
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],0] FOR btl.openib.1.9
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 1005060097
[cn2:01045] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 1005060097
[cn2:01045] [[15336,1],2] store:peer:modex: completed modex entry for proc [[15336,1],0]
[cn2:01045] [[15336,1],2] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],1]
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],1] FOR MPI_THREAD_LEVEL
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 5300027393
[cn2:01045] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 5300027393
[cn2:01045] [[15336,1],2] STORING MODEX DATA FROM [[15336,1],1] FOR btl.openib.1.9
[cn2:01045] storing data in NONPEER dstore
[cn2:01045] dstore:hash:store storing data for proc 5300027393
[cn2:01045] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 5300027393
[cn2:01045] [[15336,1],2] store:peer:modex: completed modex entry for proc [[15336,1],1]
[cn2:01045] [[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 1005060097
[cn3:21419] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 1005060097
[cn3:21419] [[15336,1],4] store:peer:modex: completed modex entry for proc [[15336,1],0]
[cn3:21419] [[15336,1],4] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],1]
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],1] FOR MPI_THREAD_LEVEL
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 5300027393
[cn3:21419] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 5300027393
[cn3:21419] [[15336,1],4] STORING MODEX DATA FROM [[15336,1],1] FOR btl.openib.1.9
[cn3:21419] storing data in NONPEER dstore
[cn3:21419] dstore:hash:store storing data for proc 5300027393
[cn3:21419] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 5300027393
[cn3:21419] [[15336,1],4] store:peer:modex: completed modex entry for proc [[15336,1],1]
[cn3:21419] [[15336,1],4] store:peer:modex NO MODEX RELEASE CBFUNC
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1655 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 377
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 377 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1651 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 377 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
ty[OPAL_UINT16] for proc 39659765761
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],9] for orte.hostname
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key orte.hostname on proc 39659765761
[cn2:01046] check:select: modex not reqd
[cn2:01046] mca: bml: Using self btl to [[15336,1],3] on node cn2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21420] [[15336,1],5]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21420] [[15336,1],5]:tcp:recv:handler CONNECTED
[cn3:21420] [[15336,1],5]:tcp:recv:handler allocate new recv msg
[cn3:21420] [[15336,1],5]:tcp:recv:handler read hdr
[cn3:21420] [[15336,1],5]:tcp:recv:handler allocate data region of size 1300
[cn3:21420] [[15336,1],5] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1300 BYTES FOR DEST [[15336,1],5] TAG 30
[cn3:21420] [[15336,1],5] DELIVERING TO RML
[cn3:21420] [[15336,1],5] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] [[15336,1],5] message received 1300 bytes from [[15336,0],2] for tag 30
[cn3:21420] [[15336,1],5] grpcomm:base:receive processing collective return for id 0 recvd from [[15336,0],2]
[cn3:21420] [[15336,1],5] CHECKING COLL id 0
[cn3:21420] [[15336,1],5] STORING MODEX DATA
[cn3:21420] [[15336,1],5] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],8]
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],8] FOR MPI_THREAD_LEVEL
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 35364798465
[cn3:21420] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 35364798465
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],8] FOR btl.openib.1.9
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 35364798465
[cn3:21420] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 35364798465
[cn3:21420] [[15336,1],5] store:peer:modex: completed modex entry for proc [[15336,1],8]
[cn3:21420] [[15336,1],5] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],9]
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],9] FOR MPI_THREAD_LEVEL
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 39659765761
[cn3:21420] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 39659765761
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],9] FOR btl.openib.1.9
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 39659765761
[cn3:21420] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 39659765761
[cn3:21420] [[15336,1],5] store:peer:modex: completed modex entry for proc [[15336,1],9]
[cn3:21420] [[15336,1],5] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],4]
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],4] FOR MPI_THREAD_LEVEL
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 18184929281
[cn3:21420] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 18184929281
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],4] FOR btl.openib.1.9
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 18184929281
[cn3:21420] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 18184929281
[cn3:21420] [[15336,1],5] store:peer:modex: completed modex entry for proc [[15336,1],4]
[cn3:21420] [[15336,1],5] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],5]
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],5] FOR MPI_THREAD_LEVEL
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],5] FOR btl.openib.1.9
[cn3:21420] [[15336,1],5] store:peer:modex: completed modex entry for proc [[15336,1],5]
[cn3:21420] [[15336,1],5] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],2]
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],2] FOR MPI_THREAD_LEVEL
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 9594994689
[cn3:21420] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 9594994689
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],2] FOR btl.openib.1.9
[cn3:21420] storing data in NONPEER dstore
[[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 84
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 84 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 84 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[15336,1],2] store:peer:modex NO MODEX RELEASE CBFUNC
[cn5:14174] [[15336,1],8]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14174] [[15336,1],8]:tcp:recv:handler CONNECTED
[cn5:14174] [[15336,1],8]:tcp:recv:handler allocate new recv msg
[cn5:14174] [[15336,1],8]:tcp:recv:handler read hdr
[cn5:14174] [[15336,1],8]:tcp:recv:handler allocate data region of size 1300
[cn5:14174] [[15336,1],8] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1300 BYTES FOR DEST [[15336,1],8] TAG 30
[cn5:14174] [[15336,1],8] DELIVERING TO RML
[cn5:14174] [[15336,1],8] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14174] [[15336,1],8] message received 1300 bytes from [[15336,0],4] for tag 30
[cn5:14174] [[15336,1],8] grpcomm:base:receive processing collective return for id 0 recvd from [[15336,0],4]
[cn5:14174] [[15336,1],8] CHECKING COLL id 0
[cn5:14174] [[15336,1],8] STORING MODEX DATA
[cn5:14174] [[15336,1],8] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],8]
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],8] FOR MPI_THREAD_LEVEL
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],8] FOR btl.openib.1.9
[cn5:14174] [[15336,1],8] store:peer:modex: completed modex entry for proc [[15336,1],8]
[cn5:14174] [[15336,1],8] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],9]
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],9] FOR MPI_THREAD_LEVEL
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 39659765761
[cn5:14174] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 39659765761
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],9] FOR btl.openib.1.9
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 39659765761
[cn5:14174] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 39659765761
[cn5:14174] [[15336,1],8] store:peer:modex: completed modex entry for proc [[15336,1],9]
[cn5:14174] [[15336,1],8] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],4]
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],4] FOR MPI_THREAD_LEVEL
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 18184929281
[cn5:14174] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 18184929281
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],4] FOR btl.openib.1.9
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 18184929281
[cn5:14174] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 18184929281
[cn5:14174] [[15336,1],8] store:peer:modex: completed modex entry for proc [[15336,1],4]
[cn5:14174] [[15336,1],8] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],5]
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],5] FOR MPI_THREAD_LEVEL
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 22479896577
[cn5:14174] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 22479896577
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],5] FOR btl.openib.1.9
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 22479896577
[cn5:14174] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 22479896577
[cn5:14174] [[15336,1],8] store:peer:modex: completed modex entry for proc [[15336,1],5]
[cn5:14174] [[15336,1],8] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],2]
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],2] FOR MPI_THREAD_LEVEL
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 9594994689
[cn5:14174] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 9594994689
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],2] FOR btl.openib.1.9
[cn5:14174] storing data in NONPEER dstore
[[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key opal.locality on proc 1005060097
[cn4:17831] dstore_hash:fetch key opal.locality for proc 1005060097 not found
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 1005060097
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostid on proc 1005060097
[cn4:17831] dstore_hash:fetch key orte.hostid for proc 1005060097 not found
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 1005060097
[cn4:17831] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 1005060097
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],0] for orte.hostname
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 1005060097
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key opal.locality on proc 5300027393
[cn4:17831] dstore_hash:fetch key opal.locality for proc 5300027393 not found
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 5300027393
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostid on proc 5300027393
[cn4:17831] dstore_hash:fetch key orte.hostid for proc 5300027393 not found
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 5300027393
[cn4:17831] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 5300027393
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],1] for orte.hostname
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 5300027393
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key opal.locality on proc 9594994689
[cn4:17831] dstore_hash:fetch key opal.locality for proc 9594994689 not found
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 9594994689
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostid on proc 9594994689
[cn4:17831] dstore_hash:fetch key orte.hostid for proc 9594994689 not found
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 9594994689
[cn4:17831] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 9594994689
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],2] for orte.hostname
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 9594994689
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key opal.locality on proc 13889961985
[cn4:17831] dstore_hash:fetch key opal.locality for proc 13889961985 not found
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 13889961985
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostid on proc 13889961985
[cn4:17831] dstore_hash:fetch key orte.hostid for proc 13889961985 not found
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 13889961985
[cn4:17831] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 13889961985
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],3] for orte.hostname
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 13889961985
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key opal.locality on proc 18184929281
[cn4:17831] dstore_hash:fetch key opal.locality for proc 18184929281 not found
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.daemon.vpid on proc[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key opal.locality on proc 1005060097
[cn3:21419] dstore_hash:fetch key opal.locality for proc 1005060097 not found
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 1005060097
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostid on proc 1005060097
[cn3:21419] dstore_hash:fetch key orte.hostid for proc 1005060097 not found
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 1005060097
[cn3:21419] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 1005060097
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],0] for orte.hostname
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 1005060097
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key opal.locality on proc 5300027393
[cn3:21419] dstore_hash:fetch key opal.locality for proc 5300027393 not found
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 5300027393
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostid on proc 5300027393
[cn3:21419] dstore_hash:fetch key orte.hostid for proc 5300027393 not found
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 5300027393
[cn3:21419] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 5300027393
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],1] for orte.hostname
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 5300027393
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key opal.locality on proc 9594994689
[cn3:21419] dstore_hash:fetch key opal.locality for proc 9594994689 not found
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 9594994689
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostid on proc 9594994689
[cn3:21419] dstore_hash:fetch key orte.hostid for proc 9594994689 not found
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 9594994689
[cn3:21419] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 9594994689
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],2] for orte.hostname
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 9594994689
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key opal.locality on proc 13889961985
[cn3:21419] dstore_hash:fetch key opal.locality for proc 13889961985 not found
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 13889961985
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostid on proc 13889961985
[cn3:21419] dstore_hash:fetch key orte.hostid for proc 13889961985 not found
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 13889961985
[cn3:21419] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 13889961985
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],3] for orte.hostname
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 13889961985
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key opal.locality on proc 22479896577
[cn3:21419] dstore_hash:fetch key opal.locality for proc 22479896577 not found
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.daemon.vpid on proc[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 509
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 509 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 950 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2039 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 509 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:175:btl_openib_async_commandh] Got cmd 0
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:178:btl_openib_async_commandh] Got fd 20
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:180:btl_openib_async_commandh] Adding device [20] to async event poll[1]
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key opal.locality on proc 1005060097
[cn5:14175] dstore_hash:fetch key opal.locality for proc 1005060097 not found
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 1005060097
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostid on proc 1005060097
[cn5:14175] dstore_hash:fetch key orte.hostid for proc 1005060097 not found
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 1005060097
[cn5:14175] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 1005060097
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],0] for orte.hostname
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 1005060097
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key opal.locality on proc 5300027393
[cn5:14175] dstore_hash:fetch key opal.locality for proc 5300027393 not found
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 5300027393
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostid on proc 5300027393
[cn5:14175] dstore_hash:fetch key orte.hostid for proc 5300027393 not found
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 5300027393
[cn5:14175] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 5300027393
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],1] for orte.hostname
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 5300027393
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key opal.locality on proc 9594994689
[cn5:14175] dstore_hash:fetch key opal.locality for proc 9594994689 not found
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 9594994689
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostid on proc 9594994689
[cn5:14175] dstore_hash:fetch key orte.hostid for proc 9594994689 not found
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 9594994689
[cn5:14175] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 9594994689
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],2] for orte.hostname
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 9594994689
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key opal.locality on proc 13889961985
[cn5:14175] dstore_hash:fetch key opal.locality for proc 13889961985 not found
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 13889961985
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostid on proc 13889961985
[cn5:14175] dstore_hash:fetch key orte.hostid for proc 13889961985 not found
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 13889961985
[cn5:14175] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 13889961985
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],3] for orte.hostname
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 13889961985
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key opal.locality on proc 18184929281
[cn5:14175] dstore_hash:fetch key opal.locality for proc 18184929281 not found
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.daemon.vpid on proc7832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],2] FOR MPI_THREAD_LEVEL
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 9594994689
[cn4:17832] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 9594994689
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],2] FOR btl.openib.1.9
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 9594994689
[cn4:17832] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 9594994689
[cn4:17832] [[15336,1],7] store:peer:modex: completed modex entry for proc [[15336,1],2]
[cn4:17832] [[15336,1],7] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],3]
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],3] FOR MPI_THREAD_LEVEL
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 13889961985
[cn4:17832] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 13889961985
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],3] FOR btl.openib.1.9
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 13889961985
[cn4:17832] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 13889961985
[cn4:17832] [[15336,1],7] store:peer:modex: completed modex entry for proc [[15336,1],3]
[cn4:17832] [[15336,1],7] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],6]
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],6] FOR MPI_THREAD_LEVEL
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 26774863873
[cn4:17832] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 26774863873
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],6] FOR btl.openib.1.9
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 26774863873
[cn4:17832] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 26774863873
[cn4:17832] [[15336,1],7] store:peer:modex: completed modex entry for proc [[15336,1],6]
[cn4:17832] [[15336,1],7] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],7]
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],7] FOR MPI_THREAD_LEVEL
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],7] FOR btl.openib.1.9
[cn4:17832] [[15336,1],7] store:peer:modex: completed modex entry for proc [[15336,1],7]
[cn4:17832] [[15336,1],7] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],0]
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],0] FOR MPI_THREAD_LEVEL
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 1005060097
[cn4:17832] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 1005060097
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],0] FOR btl.openib.1.9
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 1005060097
[cn4:17832] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 1005060097
[cn4:17832] [[15336,1],7] store:peer:modex: completed modex entry for proc [[15336,1],0]
[cn4:17832] [[15336,1],7] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],1]
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],1] FOR MPI_THREAD_LEVEL
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 5300027393
[cn4:17832] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 5300027393
[cn4:17832] [[15336,1],7] STORING MODEX DATA FROM [[15336,1],1] FOR btl.openib.1.9
[cn4:17832] storing data in NONPEER dstore
[cn4:17832] dstore:hash:store storing data for proc 5300027393
[cn4:17832] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 5300027393
[cn4:17832] [[15336,1],7] store:peer:modex: completed modex entry for proc [[15336,1],1]
[cn4:17832] [[153[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 77
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 77 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 77 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
cn5:14174] dstore:hash:store storing data for proc 9594994689
[cn5:14174] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 9594994689
[cn5:14174] [[15336,1],8] store:peer:modex: completed modex entry for proc [[15336,1],2]
[cn5:14174] [[15336,1],8] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],3]
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],3] FOR MPI_THREAD_LEVEL
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 13889961985
[cn5:14174] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 13889961985
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],3] FOR btl.openib.1.9
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 13889961985
[cn5:14174] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 13889961985
[cn5:14174] [[15336,1],8] store:peer:modex: completed modex entry for proc [[15336,1],3]
[cn5:14174] [[15336,1],8] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],6]
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],6] FOR MPI_THREAD_LEVEL
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 26774863873
[cn5:14174] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 26774863873
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],6] FOR btl.openib.1.9
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 26774863873
[cn5:14174] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 26774863873
[cn5:14174] [[15336,1],8] store:peer:modex: completed modex entry for proc [[15336,1],6]
[cn5:14174] [[15336,1],8] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],7]
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],7] FOR MPI_THREAD_LEVEL
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 31069831169
[cn5:14174] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 31069831169
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],7] FOR btl.openib.1.9
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 31069831169
[cn5:14174] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 31069831169
[cn5:14174] [[15336,1],8] store:peer:modex: completed modex entry for proc [[15336,1],7]
[cn5:14174] [[15336,1],8] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],0]
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],0] FOR MPI_THREAD_LEVEL
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 1005060097
[cn5:14174] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 1005060097
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],0] FOR btl.openib.1.9
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 1005060097
[cn5:14174] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 1005060097
[cn5:14174] [[15336,1],8] store:peer:modex: completed modex entry for proc [[15336,1],0]
[cn5:14174] [[15336,1],8] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],1]
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],1] FOR MPI_THREAD_LEVEL
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 5300027393
[cn5:14174] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 5300027393
[cn5:14174] [[15336,1],8] STORING MODEX DATA FROM [[15336,1],1] FOR btl.openib.1.9
[cn5:14174] storing data in NONPEER dstore
[cn5:14174] dstore:hash:store storing data for proc 5300027393
[cn5:14174] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 5300027393
[cn5:14174] [[15336,1],8] store:peer:modex: completed modex entry for proc [[15336,1],1]
[cn5:14174] [[153 18184929281
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostid on proc 18184929281
[cn4:17831] dstore_hash:fetch key orte.hostid for proc 18184929281 not found
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 18184929281
[cn4:17831] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 18184929281
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],4] for orte.hostname
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 18184929281
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key opal.locality on proc 22479896577
[cn4:17831] dstore_hash:fetch key opal.locality for proc 22479896577 not found
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 22479896577
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostid on proc 22479896577
[cn4:17831] dstore_hash:fetch key orte.hostid for proc 22479896577 not found
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 22479896577
[cn4:17831] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 22479896577
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],5] for orte.hostname
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 22479896577
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key opal.locality on proc 31069831169
[cn4:17831] dstore_hash:fetch key opal.locality for proc 31069831169 not found
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 31069831169
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key opal.cpuset on proc 31069831169
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 31069831169
[cn4:17831] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 31069831169
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],7] for orte.hostname
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 31069831169
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key opal.locality on proc 35364798465
[cn4:17831] dstore_hash:fetch key opal.locality for proc 35364798465 not found
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 35364798465
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostid on proc 35364798465
[cn4:17831] dstore_hash:fetch key orte.hostid for proc 35364798465 not found
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 35364798465
[cn4:17831] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 35364798465
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],8] for orte.hostname
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 35364798465
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key opal.locality on proc 39659765761
[cn4:17831] dstore_hash:fetch key opal.locality for proc 39659765761 not found
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 39659765761
[cn4:17831] fetching data from INTERNAL dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostid on proc 39659765761
[cn4:17831] dstore_hash:fetch key orte.hostid for proc 39659765761 not found
[cn4:17831] storing data in INTERNAL dstore
[cn4:17831] dstore:hash:store storing data for proc 39659765761
[cn4:17831] dstore:hash:store: storing key opal.localit[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 112
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1152 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 112 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 112 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01045] dstore:hash:fetch: searching for key opal.locality on proc 1005060097
 18184929281
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostid on proc 18184929281
[cn5:14175] dstore_hash:fetch key orte.hostid for proc 18184929281 not found
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 18184929281
[cn5:14175] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 18184929281
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],4] for orte.hostname
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 18184929281
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key opal.locality on proc 22479896577
[cn5:14175] dstore_hash:fetch key opal.locality for proc 22479896577 not found
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 22479896577
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostid on proc 22479896577
[cn5:14175] dstore_hash:fetch key orte.hostid for proc 22479896577 not found
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 22479896577
[cn5:14175] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 22479896577
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],5] for orte.hostname
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 22479896577
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key opal.locality on proc 26774863873
[cn5:14175] dstore_hash:fetch key opal.locality for proc 26774863873 not found
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 26774863873
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostid on proc 26774863873
[cn5:14175] dstore_hash:fetch key orte.hostid for proc 26774863873 not found
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 26774863873
[cn5:14175] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 26774863873
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],6] for orte.hostname
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 26774863873
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key opal.locality on proc 31069831169
[cn5:14175] dstore_hash:fetch key opal.locality for proc 31069831169 not found
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 31069831169
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostid on proc 31069831169
[cn5:14175] dstore_hash:fetch key orte.hostid for proc 31069831169 not found
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 31069831169
[cn5:14175] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 31069831169
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],7] for orte.hostname
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 31069831169
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key opal.locality on proc 35364798465
[cn5:14175] dstore_hash:fetch key opal.locality for proc 35364798465 not found
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 35364798465
[cn5:14175] fetching data from INTERNAL dstore
[cn5:14175] dstore:hash:fetch: searching for key opal.cpuset on proc 35364798465
[cn5:14175] storing data in INTERNAL dstore
[cn5:14175] dstore:hash:store storing data for proc 35364798465
[cn5:14175] dstore:hash:store: storing key opal.localit36,1],7] store:peer:modex NO MODEX RELEASE CBFUNC
cn3:21420] dstore:hash:store storing data for proc 9594994689
[cn3:21420] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 9594994689
[cn3:21420] [[15336,1],5] store:peer:modex: completed modex entry for proc [[15336,1],2]
[cn3:21420] [[15336,1],5] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],3]
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],3] FOR MPI_THREAD_LEVEL
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 13889961985
[cn3:21420] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 13889961985
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],3] FOR btl.openib.1.9
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 13889961985
[cn3:21420] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 13889961985
[cn3:21420] [[15336,1],5] store:peer:modex: completed modex entry for proc [[15336,1],3]
[cn3:21420] [[15336,1],5] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],6]
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],6] FOR MPI_THREAD_LEVEL
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 26774863873
[cn3:21420] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 26774863873
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],6] FOR btl.openib.1.9
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 26774863873
[cn3:21420] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 26774863873
[cn3:21420] [[15336,1],5] store:peer:modex: completed modex entry for proc [[15336,1],6]
[cn3:21420] [[15336,1],5] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],7]
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],7] FOR MPI_THREAD_LEVEL
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 31069831169
[cn3:21420] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 31069831169
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],7] FOR btl.openib.1.9
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 31069831169
[cn3:21420] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 31069831169
[cn3:21420] [[15336,1],5] store:peer:modex: completed modex entry for proc [[15336,1],7]
[cn3:21420] [[15336,1],5] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],0]
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],0] FOR MPI_THREAD_LEVEL
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 1005060097
[cn3:21420] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 1005060097
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],0] FOR btl.openib.1.9
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 1005060097
[cn3:21420] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 1005060097
[cn3:21420] [[15336,1],5] store:peer:modex: completed modex entry for proc [[15336,1],0]
[cn3:21420] [[15336,1],5] grpcomm:base:update_modex_entries: adding 2 entries for proc [[15336,1],1]
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],1] FOR MPI_THREAD_LEVEL
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 5300027393
[cn3:21420] dstore:hash:store: storing key MPI_THREAD_LEVEL[OPAL_BYTE_OBJECT] for proc 5300027393
[cn3:21420] [[15336,1],5] STORING MODEX DATA FROM [[15336,1],1] FOR btl.openib.1.9
[cn3:21420] storing data in NONPEER dstore
[cn3:21420] dstore:hash:store storing data for proc 5300027393
[cn3:21420] dstore:hash:store: storing key btl.openib.1.9[OPAL_BYTE_OBJECT] for proc 5300027393
[cn3:21420] [[15336,1],5] store:peer:modex: completed modex entry for proc [[15336,1],1]
[cn3:21420] [[153[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 108
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 108 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 509
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 509 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 108 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 509 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 364
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 364 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 377 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 509
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 509 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 376
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 84 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 376 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 509 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01045] dstore_hash:fetch key opal.locality for proc 1005060097 not found
36,1],8] store:peer:modex NO MODEX RELEASE CBFUNC
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:175:btl_openib_async_commandh] Got cmd 0
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:178:btl_openib_async_commandh] Got fd 21
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:180:btl_openib_async_commandh] Adding device [21] to async event poll[1]
 22479896577
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key opal.cpuset on proc 22479896577
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 22479896577
[cn3:21419] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 22479896577
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],5] for orte.hostname
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 22479896577
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key opal.locality on proc 26774863873
[cn3:21419] dstore_hash:fetch key opal.locality for proc 26774863873 not found
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 26774863873
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostid on proc 26774863873
[cn3:21419] dstore_hash:fetch key orte.hostid for proc 26774863873 not found
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 26774863873
[cn3:21419] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 26774863873
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],6] for orte.hostname
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 26774863873
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key opal.locality on proc 31069831169
[cn3:21419] dstore_hash:fetch key opal.locality for proc 31069831169 not found
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 31069831169
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostid on proc 31069831169
[cn3:21419] dstore_hash:fetch key orte.hostid for proc 31069831169 not found
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 31069831169
[cn3:21419] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 31069831169
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],7] for orte.hostname
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 31069831169
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key opal.locality on proc 35364798465
[cn3:21419] dstore_hash:fetch key opal.locality for proc 35364798465 not found
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 35364798465
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostid on proc 35364798465
[cn3:21419] dstore_hash:fetch key orte.hostid for proc 35364798465 not found
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 35364798465
[cn3:21419] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 35364798465
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],8] for orte.hostname
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 35364798465
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key opal.locality on proc 39659765761
[cn3:21419] dstore_hash:fetch key opal.locality for proc 39659765761 not found
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 39659765761
[cn3:21419] fetching data from INTERNAL dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostid on proc 39659765761
[cn3:21419] dstore_hash:fetch key orte.hostid for proc 39659765761 not found
[cn3:21419] storing data in INTERNAL dstore
[cn3:21419] dstore:hash:store storing data for proc 39659765761
[cn3:21419] dstore:hash:store: storing key opal.localit[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 364 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 509 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 376 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1838
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1838 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 376
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 376 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 77
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 77 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 376
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 376 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 1005060097
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostid on proc 1005060097
[cn2:01045] dstore_hash:fetch key orte.hostid for proc 1005060097 not found
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:175:btl_openib_async_commandh] Got cmd 0
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:178:btl_openib_async_commandh] Got fd 21
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:180:btl_openib_async_commandh] Adding device [21] to async event poll[1]
y[OPAL_UINT16] for proc 39659765761
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],9] for orte.hostname
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key orte.hostname on proc 39659765761
[cn4:17831] check:select: modex not reqd
[cn4:17831] mca: bml: Using self btl to [[15336,1],6] on node cn4
36,1],5] store:peer:modex NO MODEX RELEASE CBFUNC
[cn1:14209] [[15336,0],0] message received 1838 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 376 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 77 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 376 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 1005060097
[cn2:01045] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 1005060097
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],0] for orte.hostname
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 1005060097
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key opal.locality on proc 5300027393
[cn2:01045] dstore_hash:fetch key opal.locality for proc 5300027393 not found
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 5300027393
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostid on proc 5300027393
[cn2:01045] dstore_hash:fetch key orte.hostid for proc 5300027393 not found
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 5300027393
[cn2:01045] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 5300027393
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],1] for orte.hostname
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 5300027393
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key opal.locality on proc 13889961985
[cn2:01045] dstore_hash:fetch key opal.locality for proc 13889961985 not found
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 13889961985
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key opal.cpuset on proc 13889961985
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
y[OPAL_UINT16] for proc 35364798465
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],8] for orte.hostname
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key orte.hostname on proc 35364798465
[cn5:14175] check:select: modex not reqd
[cn5:14175] mca: bml: Using self btl to [[15336,1],9] on node cn5
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17832] fetching data from INTERNAL dstore
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
y[OPAL_UINT16] for proc 39659765761
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],9] for orte.hostname
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key orte.hostname on proc 39659765761
[cn3:21419] check:select: modex not reqd
[cn3:21419] mca: bml: Using self btl to [[15336,1],4] on node cn3
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1589
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1589 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 77
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 77 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 112
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 112 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 509
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 509 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1589 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 77 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 112 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 509 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1764
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1764 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 112
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 112 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 77 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2519
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 2519 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 283
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 283 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 13889961985
[cn2:01045] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 13889961985
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],3] for orte.hostname
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 13889961985
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key opal.locality on proc 18184929281
[cn2:01045] dstore_hash:fetch key opal.locality for proc 18184929281 not found
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 18184929281
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostid on proc 18184929281
[cn2:01045] dstore_hash:fetch key orte.hostid for proc 18184929281 not found
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 18184929281
[cn2:01045] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 18184929281
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],4] for orte.hostname
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 18184929281
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key opal.locality on proc 22479896577
[cn2:01045] dstore_hash:fetch key opal.locality for proc 22479896577 not found
[cn5:14174] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key opal.locality on proc 1005060097
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:175:btl_openib_async_commandh] Got cmd 0
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:178:btl_openib_async_commandh] Got fd 21
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:180:btl_openib_async_commandh] Adding device [21] to async event poll[1]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1764 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 112 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 2519 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 283 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2267
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2267 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 234
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 234 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1999
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1999 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1995
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1995 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 22479896577
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostid on proc 22479896577
[cn2:01045] dstore_hash:fetch key orte.hostid for proc 22479896577 not found
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 22479896577
[cn2:01045] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 22479896577
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],5] for orte.hostname
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 22479896577
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key opal.locality on proc 26774863873
[cn2:01045] dstore_hash:fetch key opal.locality for proc 26774863873 not found
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 26774863873
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostid on proc 26774863873
[cn2:01045] dstore_hash:fetch key orte.hostid for proc 26774863873 not found
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 26774863873
[cn2:01045] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 26774863873
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],6] for orte.hostname
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 26774863873
[cn2:01045] fetching data from INTERNAL dstore
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14174] dstore:hash:fetch: searching for key opal.locality on proc 1005060097
[cn4:17832] dstore_hash:fetch key opal.locality for proc 1005060097 not found
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 1005060097
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostid on proc 1005060097
[cn4:17832] dstore_hash:fetch key orte.hostid for proc 1005060097 not found
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 1005060097
[cn4:17832] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 1005060097
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],0] for orte.hostname
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 1005060097
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key opal.locality on proc 5300027393
[cn4:17832] dstore_hash:fetch key opal.locality for proc 5300027393 not found
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 5300027393
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostid on proc 5300027393
[cn4:17832] dstore_hash:fetch key orte.hostid for proc 5300027393 not found
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 5300027393
[cn4:17832] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 5300027393
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],1] for orte.hostname
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 5300027393
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key opal.locality on proc 9594994689
[cn4:17832] dstore_hash:fetch key opal.locality for proc 9594994689 not found
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 9594994689
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostid on proc 9594994689
[cn4:17832] dstore_hash:fetch key orte.hostid for proc 9594994689 not found
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 9594994689
[cn4:17832] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 9594994689
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key opal.locality on proc 1005060097
[cn3:21420] dstore_hash:fetch key opal.locality for proc 1005060097 not found
[cn3:21420] fetching data from NONPEER dstore
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 2267 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 234 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 1999 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1995 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 590
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 590 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1956
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1956 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 112 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1620
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1620 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2049
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 2049 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] dstore:hash:fetch: searching for key opal.locality on proc 31069831169
[cn2:01045] dstore_hash:fetch key opal.locality for proc 31069831169 not found
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 31069831169
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostid on proc 31069831169
[cn2:01045] dstore_hash:fetch key orte.hostid for proc 31069831169 not found
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 31069831169
[cn2:01045] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 31069831169
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],7] for orte.hostname
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 31069831169
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key opal.locality on proc 35364798465
[cn2:01045] dstore_hash:fetch key opal.locality for proc 35364798465 not found
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 35364798465
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostid on proc 35364798465
[cn2:01045] dstore_hash:fetch key orte.hostid for proc 35364798465 not found
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 35364798465
[cn2:01045] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 35364798465
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],8] for orte.hostname
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 35364798465
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01045] dstore:hash:fetch: searching for key opal.locality on proc 39659765761
[cn2:01045] dstore_hash:fetch key opal.locality for proc 39659765761 not found
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 39659765761
[cn2:01045] fetching data from INTERNAL dstore
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],0] for btl.openib.1.9
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 1005060097
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],2] for orte.hostname
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 9594994689
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key opal.locality on proc 13889961985
[cn4:17832] dstore_hash:fetch key opal.locality for proc 13889961985 not found
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 13889961985
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostid on proc 13889961985
[cn4:17832] dstore_hash:fetch key orte.hostid for proc 13889961985 not found
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 13889961985
[cn4:17832] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 13889961985
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],3] for orte.hostname
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 13889961985
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key opal.locality on proc 18184929281
[cn4:17832] dstore_hash:fetch key opal.locality for proc 18184929281 not found
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 18184929281
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostid on proc 18184929281
[cn4:17832] dstore_hash:fetch key orte.hostid for proc 18184929281 not found
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 18184929281
[cn4:17832] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 18184929281
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],4] for orte.hostname
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21420] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 1005060097
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostid on proc 1005060097
[cn3:21420] dstore_hash:fetch key orte.hostid for proc 1005060097 not found
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 1005060097
[cn3:21420] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 1005060097
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],0] for orte.hostname
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 1005060097
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key opal.locality on proc 5300027393
[cn3:21420] dstore_hash:fetch key opal.locality for proc 5300027393 not found
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 5300027393
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostid on proc 5300027393
[cn3:21420] dstore_hash:fetch key orte.hostid for proc 5300027393 not found
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 5300027393
[cn3:21420] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 5300027393
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],1] for orte.hostname
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 5300027393
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key opal.locality on proc 9594994689
[cn3:21420] dstore_hash:fetch key opal.locality for proc 9594994689 not found
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 9594994689
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 590 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 1956 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 1620 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 2049 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 108 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 438
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 438 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1545
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1545 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1086
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1086 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01045] dstore:hash:fetch: searching for key orte.hostid on proc 39659765761
[cn2:01045] dstore_hash:fetch key orte.hostid for proc 39659765761 not found
[cn2:01045] storing data in INTERNAL dstore
[cn2:01045] dstore:hash:store storing data for proc 39659765761
[cn2:01045] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 39659765761
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],9] for orte.hostname
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key orte.hostname on proc 39659765761
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37181 10003 (offset now 38)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 18184929281
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key opal.locality on proc 22479896577
[cn4:17832] dstore_hash:fetch key opal.locality for proc 22479896577 not found
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 22479896577
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostid on proc 22479896577
[cn4:17832] dstore_hash:fetch key orte.hostid for proc 22479896577 not found
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 22479896577
[cn4:17832] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 22479896577
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],5] for orte.hostname
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 22479896577
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key opal.locality on proc 26774863873
[cn4:17832] dstore_hash:fetch key opal.locality for proc 26774863873 not found
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 26774863873
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key opal.cpuset on proc 26774863873
[cn4:17832] storing data in INTERNAL dstore
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostid on proc 9594994689
[cn3:21420] dstore_hash:fetch key orte.hostid for proc 9594994689 not found
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 9594994689
[cn3:21420] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 9594994689
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],2] for orte.hostname
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 9594994689
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key opal.locality on proc 13889961985
[cn3:21420] dstore_hash:fetch key opal.locality for proc 13889961985 not found
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 13889961985
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostid on proc 13889961985
[cn3:21420] dstore_hash:fetch key orte.hostid for proc 13889961985 not found
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 13889961985
[cn3:21420] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 13889961985
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],3] for orte.hostname
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 13889961985
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key opal.locality on proc 18184929281
[cn3:21420] dstore_hash:fetch key opal.locality for proc 18184929281 not found
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 18184929281
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key opal.cpuset on proc 18184929281
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 438 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1545 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 1086 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01045] check:select: modex not reqd
[cn2:01045] mca: bml: Using self btl to [[15336,1],2] on node cn2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],1] for btl.openib.1.9
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 5300027393
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],2] for btl.openib.1.9
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 9594994689
[cn4:17832] dstore:hash:store storing data for proc 26774863873
[cn4:17832] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 26774863873
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],6] for orte.hostname
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 26774863873
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key opal.locality on proc 35364798465
[cn4:17832] dstore_hash:fetch key opal.locality for proc 35364798465 not found
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 35364798465
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostid on proc 35364798465
[cn4:17832] dstore_hash:fetch key orte.hostid for proc 35364798465 not found
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 35364798465
[cn4:17832] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 35364798465
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],8] for orte.hostname
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 35364798465
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key opal.locality on proc 39659765761
[cn4:17832] dstore_hash:fetch key opal.locality for proc 39659765761 not found
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 18184929281
[cn3:21420] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 18184929281
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],4] for orte.hostname
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 18184929281
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key opal.locality on proc 26774863873
[cn3:21420] dstore_hash:fetch key opal.locality for proc 26774863873 not found
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 26774863873
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostid on proc 26774863873
[cn3:21420] dstore_hash:fetch key orte.hostid for proc 26774863873 not found
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 26774863873
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 364 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 509
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 509 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 509 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3195
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 376 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 3195 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 769
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 769 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2137
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 2137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 509 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 3195 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 769 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1838 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 2137 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 376 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 509 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 234
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 234 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 440
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 440 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 509 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1061
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1061 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:175:btl_openib_async_commandh] Got cmd 0
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:178:btl_openib_async_commandh] Got fd 21
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:180:btl_openib_async_commandh] Adding device [21] to async event poll[1]
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37182 10003 (offset now 38)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] [cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 39659765761
[cn4:17832] fetching data from INTERNAL dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostid on proc 39659765761
[cn4:17832] dstore_hash:fetch key orte.hostid for proc 39659765761 not found
[cn4:17832] storing data in INTERNAL dstore
[cn4:17832] dstore:hash:store storing data for proc 39659765761
[cn4:17832] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 39659765761
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],9] for orte.hostname
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key orte.hostname on proc 39659765761
[cn3:21420] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 26774863873
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],6] for orte.hostname
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 26774863873
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key opal.locality on proc 31069831169
[cn3:21420] dstore_hash:fetch key opal.locality for proc 31069831169 not found
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 31069831169
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostid on proc 31069831169
[cn3:21420] dstore_hash:fetch key orte.hostid for proc 31069831169 not found
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 31069831169
[cn3:21420] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 31069831169
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],7] for orte.hostname
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 31069831169
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key opal.locality on proc 35364798465
[cn3:21420] dstore_hash:fetch key opal.locality for proc 35364798465 not found
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 35364798465
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostid on proc 35364798465
[cn3:21420] dstore_hash:fetch key orte.hostid for proc 35364798465 not found
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 35364798465
[cn3:21420] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 35364798465
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],8] for orte.hostname
[cn3:21420] fetching data from NONPEER dstore
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 234 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 77 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 440 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 376 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 1061 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2491
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2491 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 283 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3365
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 3365 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1589 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 509
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 112 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 509 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],0] for btl.openib.1.9
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 1005060097
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],3] for btl.openib.1.9
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 13889961985
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],4] for btl.openib.1.9
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 18184929281
[cn4:17832] check:select: modex not reqd
[cn4:17832] mca: bml: Using self btl to [[15336,1],7] on node cn4
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 35364798465
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key opal.locality on proc 39659765761
[cn3:21420] dstore_hash:fetch key opal.locality for proc 39659765761 not found
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 39659765761
[cn3:21420] fetching data from INTERNAL dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostid on proc 39659765761
[cn3:21420] dstore_hash:fetch key orte.hostid for proc 39659765761 not found
[cn3:21420] storing data in INTERNAL dstore
[cn3:21420] dstore:hash:store storing data for proc 39659765761
[cn3:21420] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 39659765761
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],9] for orte.hostname
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key orte.hostname on proc 39659765761
[cn1:14209] [[15336,0],0] message received 2491 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 3365 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1764 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 509 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 77 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 438
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 438 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3160
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1995 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 3160 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2267 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1667
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1667 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 509
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 509 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37181 10003 (offset now 38)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)[cn2:01042] [[15336,0],1] routing direct to the HNP
unpacked btl 0: cpc 0: blob unpacked 37962 10004 (offset now 38)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37963 10004 (offset now 38)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:175:btl_openib_async_commandh] Got cmd 0
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:178:btl_openib_async_commandh] Got fd 20
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:180:btl_openib_async_commandh] Adding device [20] to async event poll[1]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21420] check:select: modex not reqd
[cn3:21420] mca: bml: Using self btl to [[15336,1],5] on node cn3
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 112 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 438 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 3160 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2049 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 1667 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 509 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2096
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 590 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2096 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 440
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2519 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 440 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],1] for btl.openib.1.9
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 5300027393
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],2] for btl.openib.1.9
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 9594994689
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33170 10007 (offset now 38)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33171 10007 (offset now 38)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],0] for btl.openib.1.9
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 1005060097
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],1] for btl.openib.1.9
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 5300027393
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],2] for btl.openib.1.9
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 9594994689
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],3] for btl.openib.1.9
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 13889961985
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],4] for btl.openib.1.9
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 18184929281
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],5] for btl.openib.1.9
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 22479896577
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],7] for btl.openib.1.9
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 31069831169
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],8] for btl.openib.1.9
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 35364798465
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:175:btl_openib_async_commandh] Got cmd 0
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:178:btl_openib_async_commandh] Got fd 21
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:180:btl_openib_async_commandh] Adding device [21] to async event poll[1]
[cn1:14209] [[15336,0],0] message received 2096 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 440 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]

[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37182 10003 (offset now 38)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] [cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],5] for btl.openib.1.9
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 22479896577
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],6] for btl.openib.1.9
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 26774863873
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37181 10003 (offset now 38)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37182 10003 (offset now 38)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/b[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 440
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 440 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4036
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1086 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 4036 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 234 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1999 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1872
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1872 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 440 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 4036 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 1872 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 440
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 440 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1956 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 235
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 235 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],4] for btl.openib.1.9
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 18184929281
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],5] for btl.openib.1.9
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 22479896577
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32327 10005 (offset now 38)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32328 10005 (offset now 38)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
tl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37962 10004 (offset now 38)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37963 10004 (offset now 38)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked bt[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],0] for btl.openib.1.9
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 1005060097
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],1] for btl.openib.1.9
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 5300027393
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],2] for btl.openib.1.9
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 9594994689
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],3] for btl.openib.1.9
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 13889961985
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],5] for btl.openib.1.9
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 22479896577
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],6] for btl.openib.1.9
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 26774863873
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],7] for btl.openib.1.9
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 31069831169
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],8] for btl.openib.1.9
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 35364798465
[cn3:21419] [[15336,1],4] fetch data from [[15336,1],9] for btl.openib.1.9
[cn3:21419] fetching data from NONPEER dstore
[cn3:21419] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 39659765761
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1620 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 440 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2137 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 235 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37962 10004 (offset now 38)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33170 10007 (offset now 38)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],3][/[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],7] for btl.openib.1.9
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 31069831169
[cn5:14175] [[15336,1],9] fetch data from [[15336,1],8] for btl.openib.1.9
[cn5:14175] fetching data from NONPEER dstore
[cn5:14175] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 35364798465
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 509 BYTES ON SOCKET 9
[cn4:17831] [[15336,1],6] fetch data from [[15336,1],9] for btl.openib.1.9
[cn4:17831] fetching data from NONPEER dstore
[cn4:17831] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 39659765761
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37181 10003 (offset now 38)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37182 10003 (offset now 38)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/b[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 356
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 356 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1802
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1802 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1257
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1257 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1061 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 356 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1802 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1545 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 234 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 1257 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3117
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 3117 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 286
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 286 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 438 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],6] for btl.openib.1.9
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 26774863873
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],7] for btl.openib.1.9
[cn2:01046] fetching data from NONPEER dstore
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27543 10008 (offset now 38)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
l 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33170 10007 (offset now 38)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33171 10007 (offset now 38)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[153[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],0] for btl.openib.1.9
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 1005060097
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],1] for btl.openib.1.9
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 5300027393
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],2] for btl.openib.1.9
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 9594994689
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],3] for btl.openib.1.9
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 13889961985
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],4] for btl.openib.1.9
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 18184929281
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],6] for btl.openib.1.9
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 26774863873
[cn1:14209] [[15336,0],0] message received 3117 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 286 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2491 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 769 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1397
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1397 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 980
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 980 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 509 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33171 10007 (offset now 38)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32327 10005 (offset now 38)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14174] dstore_hash:fetch key opal.locality for proc 1005060097 not found
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 1005060097
[cn5:14174] fetching data from INTERNAL dstore
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
36,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32328 10005 (offset now 38)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27543 10008 (offset now 38)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37181 10003 (offset now 38)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37182 10003 (offset now 38)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/b[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1397 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 980 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3195 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 438 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 524
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 524 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2515
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 509 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 2515 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 234
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 234 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32328 10005 (offset now 38)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27543 10008 (offset now 38)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trun[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14174] dstore:hash:fetch: searching for key orte.hostid on proc 1005060097
[cn5:14174] dstore_hash:fetch key orte.hostid for proc 1005060097 not found
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 1005060097
[cn5:14174] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 1005060097
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],0] for orte.hostname
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 1005060097
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key opal.locality on proc 5300027393
[cn5:14174] dstore_hash:fetch key opal.locality for proc 5300027393 not found
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 5300027393
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostid on proc 5300027393
[cn5:14174] dstore_hash:fetch key orte.hostid for proc 5300027393 not found
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 5300027393
[cn5:14174] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 5300027393
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],1] for orte.hostname
/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27544 10008 (offset now 38)
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
tl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37962 10004 (offset now 38)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37963 10004 (offset now 38)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked bt[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 524 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 2515 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 440 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 234 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2096 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn2:01046] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 31069831169
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],8] for btl.openib.1.9
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 35364798465
[cn2:01046] [[15336,1],3] fetch data from [[15336,1],9] for btl.openib.1.9
[cn2:01046] fetching data from NONPEER dstore
[cn2:01046] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 39659765761
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 5300027393
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key opal.locality on proc 9594994689
[cn5:14174] dstore_hash:fetch key opal.locality for proc 9594994689 not found
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 9594994689
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostid on proc 9594994689
[cn5:14174] dstore_hash:fetch key orte.hostid for proc 9594994689 not found
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 9594994689
[cn5:14174] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 9594994689
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],2] for orte.hostname
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 9594994689
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key opal.locality on proc 13889961985
[cn5:14174] dstore_hash:fetch key opal.locality for proc 13889961985 not found
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 13889961985
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostid on proc 13889961985
[cn5:14174] dstore_hash:fetch key orte.hostid for proc 13889961985 not found
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 13889961985
[cn5:14174] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 13889961985
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],3] for orte.hostname
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 13889961985
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key opal.locality on proc 18184929281
[cn5:14174] dstore_hash:fetch key opal.locality for proc 18184929281 not found
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 18184929281
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostid on proc 18184929281
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],0] for btl.openib.1.9
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 1005060097
tl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37962 10004 (offset now 38)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37963 10004 (offset now 38)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked bt[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1558
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1558 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2202
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 2202 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3365 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1956
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1956 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3160 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 645
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 645 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 440 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 440 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 1558 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 2202 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4036 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 1956 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 440 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 645 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1802 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
k_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27544 10008 (offset now 38)
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14174] dstore_hash:fetch key orte.hostid for proc 18184929281 not found
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 18184929281
[cn5:14174] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 18184929281
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],4] for orte.hostname
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 18184929281
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key opal.locality on proc 22479896577
[cn5:14174] dstore_hash:fetch key opal.locality for proc 22479896577 not found
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 22479896577
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostid on proc 22479896577
[cn5:14174] dstore_hash:fetch key orte.hostid for proc 22479896577 not found
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 22479896577
[cn5:14174] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 22479896577
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],5] for orte.hostname
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 22479896577
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key opal.locality on proc 26774863873
[cn5:14174] dstore_hash:fetch key opal.locality for proc 26774863873 not found
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 26774863873
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostid on proc 26774863873
[cn5:14174] dstore_hash:fetch key orte.hostid for proc 26774863873 not found
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 26774863873
[cn5:14174] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 26774863873
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37181 10003 (offset now 38)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],7] for btl.openib.1.9
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 31069831169
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],8] for btl.openib.1.9
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 35364798465
[cn3:21420] [[15336,1],5] fetch data from [[15336,1],9] for btl.openib.1.9
[cn3:21420] fetching data from NONPEER dstore
[cn3:21420] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 39659765761
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1522
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1522 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 438
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 438 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 286 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1522 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 438 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1397 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],6] for orte.hostname
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 26774863873
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key opal.locality on proc 31069831169
[cn5:14174] dstore_hash:fetch key opal.locality for proc 31069831169 not found
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 31069831169
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostid on proc 31069831169
[cn5:14174] dstore_hash:fetch key orte.hostid for proc 31069831169 not found
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 31069831169
[cn5:14174] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 31069831169
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],7] for orte.hostname
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 31069831169
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key opal.locality on proc 39659765761
[cn5:14174] dstore_hash:fetch key opal.locality for proc 39659765761 not found
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.daemon.vpid on proc 39659765761
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],1] for btl.openib.1.9
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 5300027393
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],2] for btl.openib.1.9
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 9594994689
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 356 BYTES ON SOCKET 9
l 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33170 10007 (offset now 38)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32327 10005 (offset now 38)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[153[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 357
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 357 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3117 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2515 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2854
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 2854 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 524 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1558 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 357 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 2854 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2202 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14174] fetching data from INTERNAL dstore
[cn5:14174] dstore:hash:fetch: searching for key opal.cpuset on proc 39659765761
[cn5:14174] storing data in INTERNAL dstore
[cn5:14174] dstore:hash:store storing data for proc 39659765761
[cn5:14174] dstore:hash:store: storing key opal.locality[OPAL_UINT16] for proc 39659765761
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37182 10003 (offset now 38)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
l 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33171 10007 (offset now 38)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32327 10005 (offset now 38)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[153[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1522 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 233
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 233 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2984
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 2984 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 357 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 233 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 233 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 2984 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1667 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 440
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 440 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 509 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],9] for orte.hostname
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key orte.hostname on proc 39659765761

[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37962 10004 (offset now 38)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37963 10004 (offset now 38)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
36,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32328 10005 (offset now 38)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27543 10008 (offset now 38)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 440 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 980
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 980 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14174] check:select: modex not reqd
[cn5:14174] mca: bml: Using self btl to [[15336,1],8] on node cn5
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],3] for btl.openib.1.9
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 13889961985
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],4] for btl.openib.1.9
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 18184929281
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
36,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32328 10005 (offset now 38)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27543 10008 (offset now 38)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 235 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 509
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 509 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3565
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 980 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 3565 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 980 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 509 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 234 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 3565 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1956 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 234
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 235
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 235 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 980
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 980 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27544 10008 (offset now 38)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:175:btl_openib_async_commandh] Got cmd 0
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:178:btl_openib_async_commandh] Got fd 21
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_async.c:180:btl_openib_async_commandh] Adding device [21] to async event poll[1]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33170 10007 (offset now 38)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33171 10007 (offset now 38)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 235 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 980 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 438 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],5] for btl.openib.1.9
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 22479896577
/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27544 10008 (offset now 38)
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 234 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 234
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 234 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4014
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 4014 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 234 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 234 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 4014 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 234
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 234 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2854 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 151
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 151 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 440
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 440 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],0] for btl.openib.1.9
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 1005060097
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],0] for btl.openib.1.9
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 1005060097
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32327 10005 (offset now 38)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27543 10008 (offset now 38)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] [cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 234 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 151 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 440 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2141
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 2141 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],1] for btl.openib.1.9
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 5300027393
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],1] for btl.openib.1.9
[cn5:14174] fetching data from NONPEER dstore
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],6] for btl.openib.1.9
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 26774863873
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],8] for btl.openib.1.9
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 35364798465
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2984 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3565
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1872 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 3565 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3415
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 3415 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 2141 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 3565 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 3415 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
got a subnet fe80000000000000
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27544 10008 (offset now 38)
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37181 10003 (offset now 38)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37182 10003 (offset now 38)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37181 10003 (offset now 38)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37182 10003 (offset now 38)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 440
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 522
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 522 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 440 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 235
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 235 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 522 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 235 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14174] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 5300027393
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],2] for btl.openib.1.9
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 9594994689
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],3] for btl.openib.1.9
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 13889961985
[cn4:17832] [[15336,1],7] fetch data from [[15336,1],9] for btl.openib.1.9
[cn4:17832] fetching data from NONPEER dstore
[cn4:17832] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 39659765761
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 440 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3145
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 440 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],3] for btl.openib.1.9
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 13889961985
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],4] for btl.openib.1.9
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 18184929281
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2809
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2809 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 2809 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37963 10004 (offset now 38)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3565 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 440
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 235 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 440 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 440 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4014 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],5] for btl.openib.1.9
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 22479896577
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],6] for btl.openib.1.9
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 26774863873
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 440 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2141 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1257 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 3145 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 235 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 3145 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33170 10007 (offset now 38)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33171 10007 (offset now 38)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-tr[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37962 10004 (offset now 38)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 310
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 310 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2842
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 2842 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 310 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 2842 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],7] for btl.openib.1.9
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 31069831169
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],8] for btl.openib.1.9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 37963 10004 (offset now 38)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33170 10007 (offset now 38)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos [cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3256
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 3256 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 645 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 440
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 440 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 3256 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 440 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
unk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32327 10005 (offset now 38)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32328 10005 (offset now 38)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],4] for btl.openib.1.9
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 18184929281
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],5] for btl.openib.1.9
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 22479896577
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 980 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 980 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 365
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 365 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3882
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 3882 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 234 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 365 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 3882 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 35364798465
[cn2:01045] [[15336,1],2] fetch data from [[15336,1],9] for btl.openib.1.9
[cn2:01045] fetching data from NONPEER dstore
[cn2:01045] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 39659765761
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]

[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 33171 10007 (offset now 38)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32327 10005 (offset now 38)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos [cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 3728
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 3728 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 235
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 234 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 234 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 235 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 3728 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 235 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27543 10008 (offset now 38)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27544 10008 (offset now 38)
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],6] for btl.openib.1.9
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 26774863873
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 440
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 440 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 440 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],7] for btl.openib.1.9
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 31069831169
[cn5:14174] [[15336,1],8] fetch data from [[15336,1],9] for btl.openib.1.9
[cn5:14174] fetching data from NONPEER dstore
[cn5:14174] dstore:hash:fetch: searching for key btl.openib.1.9 on proc 39659765761
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 151 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 4126
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3565 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 4126 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 4126 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]

[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 32328 10005 (offset now 38)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:172:mca_btl_openib_proc_create] unpack: 1 btls
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:192:mca_btl_openib_proc_create] unpacked btl 0: modex message, offset now 26
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:198:mca_btl_openib_proc_create] unpacked btl 0: number of cpcs to follow 1 (offset now 27)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:213:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: index 3 (offset now 28)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:217:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: component udcm
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:224:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: priority 63, msg len 8 (offset now 30)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:238:mca_btl_openib_proc_create] unpacked btl 0: cpc 0: blob unpacked 27544 10008 (offset now 38)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_proc.c:255:mca_btl_openib_proc_create] unpacking done!
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:979:mca_btl_openib_add_procs] got 1 port_infos 
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:982:mca_btl_openib_add_procs] got a subnet fe80000000000000
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 104
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 104 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 104 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
/openib/btl_openib.c:985:mca_btl_openib_add_procs] Got a matching subnet!
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 234
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 234 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3415 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 234 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14175] mca: bml: Using openib btl to [[15336,1],0] on node cn1
[cn5:14175] mca: bml: Using openib btl to [[15336,1],1] on node cn1
[cn5:14175] mca: bml: Using openib btl to [[15336,1],2] on node cn2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 440 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 643
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 643 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 643 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14175] mca: bml: Using openib btl to [[15336,1],3] on node cn2
[cn5:14175] mca: bml: Using openib btl to [[15336,1],4] on node cn3
[cn5:14175] mca: bml: Using openib btl to [[15336,1],5] on node cn3
[cn5:14175] mca: bml: Using openib btl to [[15336,1],6] on node cn4
[cn5:14175] mca: bml: Using openib btl to [[15336,1],7] on node cn4
[cn5:14175] mca: bml: Using openib btl to [[15336,1],8] on node cn5
[cn5:14175] [[15336,1],9] grpcomm:bad entering barrier
[cn5:14175] [[15336,1],9] ACTIVATING GRCPCOMM OP 1 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 522 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2809 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3145 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2842 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 440 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 310 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3256 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 365 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3728 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 440 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 3882 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 235 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 440 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 4126 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 104 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 234 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 643 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,1],0] (ORIGIN [[15336,1],0]) OF 10 BYTES FOR DEST [[15336,0],0] TAG 30
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,1],1] (ORIGIN [[15336,1],1]) OF 10 BYTES FOR DEST [[15336,0],0] TAG 30
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 642
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 642 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1298
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1298 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 847
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 847 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 642
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 642 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 10 bytes from [[15336,1],0] for tag 30
[cn1:14209] [[15336,0],0] COLLECTIVE RECVD FROM [[15336,1],0]
[cn1:14209] [[15336,0],0] WORKING COLLECTIVE 1
[cn1:14209] [[15336,0],0] PROGRESSING COLLECTIVE 1
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 1
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 2
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] message received 10 bytes from [[15336,1],1] for tag 30
[cn1:14209] [[15336,0],0] COLLECTIVE RECVD FROM [[15336,1],1]
[cn1:14209] [[15336,0],0] WORKING COLLECTIVE 1
[cn1:14209] [[15336,0],0] PROGRESSING COLLECTIVE 1
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 1
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] COLLECTIVE 1 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 2
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] message received 642 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 1298 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 847 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 642 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 32
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 32
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 32
[cn1:14209] [[15336,0],0] message received 32 bytes from [[15336,0],0] for tag 32
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],0]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 1
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn2:01046] mca: bml: Using openib btl to [[15336,1],0] on node cn1
[cn2:01046] mca: bml: Using openib btl to [[15336,1],1] on node cn1
[cn2:01046] mca: bml: Using openib btl to [[15336,1],2] on node cn2
[cn2:01046] mca: bml: Using openib btl to [[15336,1],4] on node cn3
[cn2:01046] mca: bml: Using openib btl to [[15336,1],5] on node cn3
[cn2:01046] mca: bml: Using openib btl to [[15336,1],6] on node cn4
[cn2:01046] mca: bml: Using openib btl to [[15336,1],7] on node cn4
[cn2:01046] mca: bml: Using openib btl to [[15336,1],8] on node cn5
[cn2:01046] mca: bml: Using openib btl to [[15336,1],9] on node cn5
[cn5:14175] [[15336,1],9] rml_send_buffer to peer [[15336,0],4] at tag 30
[cn5:14175] [[15336,1],9] grpcomm:bad barrier underway
[cn5:14175] [[15336,1],9] rml_send_msg to peer [[15336,0],4] at tag 30
[cn5:14175] [[15336,1],9] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14175] [[15336,1],9] oob:base:send to target [[15336,0],4]
[cn5:14175] [[15336,1],9] oob:base:send known transport for peer [[15336,0],4]
[cn5:14175] [[15336,1],9] oob:tcp:send_nb to peer [[15336,0],4]:30
[cn5:14175] [[15336,1],9] tcp:send_nb to peer [[15336,0],4]
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:30
[cn5:14175] [[15336,1],9] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_nb: already connected to [[15336,0],4] - queueing for send
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_handler called to send to peer [[15336,0],4]
[cn4:17831] mca: bml: Using openib btl to [[15336,1],0] on node cn1
[cn4:17831] mca: bml: Using openib btl to [[15336,1],1] on node cn1
[cn4:17831] mca: bml: Using openib btl to [[15336,1],2] on node cn2
[cn4:17831] mca: bml: Using openib btl to [[15336,1],3] on node cn2
[cn4:17831] mca: bml: Using openib btl to [[15336,1],4] on node cn3
[cn4:17831] mca: bml: Using openib btl to [[15336,1],5] on node cn3
[cn4:17831] mca: bml: Using openib btl to [[15336,1],7] on node cn4
[cn4:17831] mca: bml: Using openib btl to [[15336,1],8] on node cn5
[cn4:17831] mca: bml: Using openib btl to [[15336,1],9] on node cn5
[cn4:17831] [[15336,1],6] grpcomm:bad entering barrier
[cn4:17831] [[15336,1],6] ACTIVATING GRCPCOMM OP 1 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn3:21419] mca: bml: Using openib btl to [[15336,1],0] on node cn1
[cn3:21419] mca: bml: Using openib btl to [[15336,1],1] on node cn1
[cn3:21419] mca: bml: Using openib btl to [[15336,1],2] on node cn2
[cn3:21419] mca: bml: Using openib btl to [[15336,1],3] on node cn2
[cn3:21419] mca: bml: Using openib btl to [[15336,1],5] on node cn3
[cn3:21419] mca: bml: Using openib btl to [[15336,1],6] on node cn4
[cn3:21419] mca: bml: Using openib btl to [[15336,1],7] on node cn4
[cn3:21419] mca: bml: Using openib btl to [[15336,1],8] on node cn5
[cn3:21419] mca: bml: Using openib btl to [[15336,1],9] on node cn5
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 235
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 235 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2428
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 2428 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2428
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 2428 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 235
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 235 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 235 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 2428 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 2428 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 235 bytes from [[15336,0],2] for tag 2
[cn2:01046] [[15336,1],3] grpcomm:bad entering barrier
[cn2:01046] [[15336,1],3] ACTIVATING GRCPCOMM OP 1 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn5:14174] mca: bml: Using openib btl to [[15336,1],0] on node cn1
[cn5:14174] mca: bml: Using openib btl to [[15336,1],1] on node cn1
[cn5:14174] mca: bml: Using openib btl to [[15336,1],2] on node cn2
[cn5:14174] mca: bml: Using openib btl to [[15336,1],3] on node cn2
[cn5:14174] mca: bml: Using openib btl to [[15336,1],4] on node cn3
[cn5:14174] mca: bml: Using openib btl to [[15336,1],5] on node cn3
[cn5:14174] mca: bml: Using openib btl to [[15336,1],6] on node cn4
[cn5:14174] mca: bml: Using openib btl to [[15336,1],7] on node cn4
[cn5:14174] mca: bml: Using openib btl to [[15336,1],9] on node cn5
[cn5:14174] [[15336,1],8] grpcomm:bad entering barrier
[cn5:14174] [[15336,1],8] ACTIVATING GRCPCOMM OP 1 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn5:14174] [[15336,1],8] rml_send_buffer to peer [[15336,0],4] at tag 30
[cn5:14174] [[15336,1],8] grpcomm:bad barrier underway
[cn5:14174] [[15336,1],8] rml_send_msg to peer [[15336,0],4] at tag 30
[cn5:14174] [[15336,1],8] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14174] [[15336,1],8] oob:base:send to target [[15336,0],4]
[cn5:14174] [[15336,1],8] oob:base:send known transport for peer [[15336,0],4]
[cn5:14174] [[15336,1],8] oob:tcp:send_nb to peer [[15336,0],4]:30
[cn5:14174] [[15336,1],8] tcp:send_nb to peer [[15336,0],4]
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:30
[cn5:14174] [[15336,1],8] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_nb: already connected to [[15336,0],4] - queueing for send
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_handler called to send to peer [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_handler SENDING TO [[15336,0],4]
[cn5:14174] [[15336,1],8] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 10 BYTES ON SOCKET 11
[cn5:14174] [[15336,1],8]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17832] mca: bml: Using openib btl to [[15336,1],0] on node cn1
[cn4:17832] mca: bml: Using openib btl to [[15336,1],1] on node cn1
[cn4:17832] mca: bml: Using openib btl to [[15336,1],2] on node cn2
[cn4:17832] mca: bml: Using openib btl to [[15336,1],3] on node cn2
[cn4:17832] mca: bml: Using openib btl to [[15336,1],4] on node cn3
[cn4:17832] mca: bml: Using openib btl to [[15336,1],5] on node cn3
[cn4:17832] mca: bml: Using openib btl to [[15336,1],6] on node cn4
[cn4:17832] mca: bml: Using openib btl to [[15336,1],8] on node cn5
[cn4:17832] mca: bml: Using openib btl to [[15336,1],9] on node cn5
[cn4:17832] [[15336,1],7] grpcomm:bad entering barrier
[cn4:17832] [[15336,1],7] ACTIVATING GRCPCOMM OP 1 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn4:17832] [[15336,1],7] rml_send_buffer to peer [[15336,0],3] at tag 30
[cn4:17832] [[15336,1],7] grpcomm:bad barrier underway
[cn4:17832] [[15336,1],7] rml_send_msg to peer [[15336,0],3] at tag 30
[cn4:17832] [[15336,1],7] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17832] [[15336,1],7] oob:base:send to target [[15336,0],3]
[cn4:17832] [[15336,1],7] oob:base:send known transport for peer [[15336,0],3]
[cn4:17832] [[15336,1],7] oob:tcp:send_nb to peer [[15336,0],3]:30
[cn4:17832] [[15336,1],7] tcp:send_nb to peer [[15336,0],3]
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:30
[cn4:17832] [[15336,1],7] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_nb: already connected to [[15336,0],3] - queueing for send
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_handler called to send to peer [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_handler SENDING TO [[15336,0],3]
[cn4:17832] [[15336,1],7] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 10 BYTES ON SOCKET 11
[cn4:17832] [[15336,1],7]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21419] [[15336,1],4] grpcomm:bad entering barrier
[cn3:21419] [[15336,1],4] ACTIVATING GRCPCOMM OP 1 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2428
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 2428 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 343
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 343 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 159
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 159 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2428
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 2428 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 2428 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 343 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 159 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 2428 bytes from [[15336,0],2] for tag 2
[cn2:01045] mca: bml: Using openib btl to [[15336,1],0] on node cn1
[cn2:01045] mca: bml: Using openib btl to [[15336,1],1] on node cn1
[cn2:01045] mca: bml: Using openib btl to [[15336,1],3] on node cn2
[cn2:01045] mca: bml: Using openib btl to [[15336,1],4] on node cn3
[cn2:01045] mca: bml: Using openib btl to [[15336,1],5] on node cn3
[cn2:01045] mca: bml: Using openib btl to [[15336,1],6] on node cn4
[cn2:01045] mca: bml: Using openib btl to [[15336,1],7] on node cn4
[cn2:01045] mca: bml: Using openib btl to [[15336,1],8] on node cn5
[cn2:01045] mca: bml: Using openib btl to [[15336,1],9] on node cn5
[cn2:01045] [[15336,1],2] grpcomm:bad entering barrier
[cn2:01045] [[15336,1],2] ACTIVATING GRCPCOMM OP 1 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn2:01045] [[15336,1],2] rml_send_buffer to peer [[15336,0],1] at tag 30
[cn2:01045] [[15336,1],2] grpcomm:bad barrier underway
[cn2:01045] [[15336,1],2] rml_send_msg to peer [[15336,0],1] at tag 30
[cn2:01045] [[15336,1],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01045] [[15336,1],2] oob:base:send to target [[15336,0],1]
[cn2:01045] [[15336,1],2] oob:base:send known transport for peer [[15336,0],1]
[cn2:01045] [[15336,1],2] oob:tcp:send_nb to peer [[15336,0],1]:30
[cn2:01045] [[15336,1],2] tcp:send_nb to peer [[15336,0],1]
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:30
[cn2:01045] [[15336,1],2] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_handler called to send to peer [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_handler SENDING TO [[15336,0],1]
[cn2:01045] [[15336,1],2] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 10 BYTES ON SOCKET 11
[cn2:01045] [[15336,1],2]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14175] [[15336,1],9] tcp:send_handler SENDING TO [[15336,0],4]
[cn5:14175] [[15336,1],9] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 10 BYTES ON SOCKET 11
[cn5:14175] [[15336,1],9]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17831] [[15336,1],6] rml_send_buffer to peer [[15336,0],3] at tag 30
[cn4:17831] [[15336,1],6] grpcomm:bad barrier underway
[cn3:21420] mca: bml: Using openib btl to [[15336,1],0] on node cn1
[cn3:21420] mca: bml: Using openib btl to [[15336,1],1] on node cn1
[cn3:21420] mca: bml: Using openib btl to [[15336,1],2] on node cn2
[cn3:21420] mca: bml: Using openib btl to [[15336,1],3] on node cn2
[cn3:21420] mca: bml: Using openib btl to [[15336,1],4] on node cn3
[cn3:21420] mca: bml: Using openib btl to [[15336,1],6] on node cn4
[cn3:21420] mca: bml: Using openib btl to [[15336,1],7] on node cn4
[cn3:21420] mca: bml: Using openib btl to [[15336,1],8] on node cn5
[cn3:21420] mca: bml: Using openib btl to [[15336,1],9] on node cn5
[cn3:21420] [[15336,1],5] grpcomm:bad entering barrier
[cn3:21420] [[15336,1],5] ACTIVATING GRCPCOMM OP 1 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn3:21420] [[15336,1],5] rml_send_buffer to peer [[15336,0],2] at tag 30
[cn3:21420] [[15336,1],5] grpcomm:bad barrier underway
[cn3:21420] [[15336,1],5] rml_send_msg to peer [[15336,0],2] at tag 30
[cn3:21420] [[15336,1],5] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21420] [[15336,1],5] oob:base:send to target [[15336,0],2]
[cn3:21420] [[15336,1],5] oob:base:send known transport for peer [[15336,0],2]
[cn3:21420] [[15336,1],5] oob:tcp:send_nb to peer [[15336,0],2]:30
[cn3:21420] [[15336,1],5] tcp:send_nb to peer [[15336,0],2]
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:30
[cn3:21420] [[15336,1],5] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_nb: already connected to [[15336,0],2] - queueing for send
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_handler called to send to peer [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_handler SENDING TO [[15336,0],2]
[cn3:21420] [[15336,1],5] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 10 BYTES ON SOCKET 11
[cn3:21420] [[15336,1],5]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 32
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 32 BYTES FOR DEST [[15336,0],0] TAG 32
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1237
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1237 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 32 bytes from [[15336,0],4] for tag 32
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],4]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 1
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 4
[cn1:14209] [[15336,0],0] message received 1237 bytes from [[15336,0],3] for tag 2
[cn4:17831] [[15336,1],6] rml_send_msg to peer [[15336,0],3] at tag 30
[cn4:17831] [[15336,1],6] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17831] [[15336,1],6] oob:base:send to target [[15336,0],3]
[cn4:17831] [[15336,1],6] oob:base:send known transport for peer [[15336,0],3]
[cn4:17831] [[15336,1],6] oob:tcp:send_nb to peer [[15336,0],3]:30
[cn4:17831] [[15336,1],6] tcp:send_nb to peer [[15336,0],3]
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:30
[cn4:17831] [[15336,1],6] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn4:17831] [[15336,1],6] tcp:send_nb: already connected to [[15336,0],3] - queueing for send
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],3]
[cn4:17831] [[15336,1],6] tcp:send_handler called to send to peer [[15336,0],3]
[cn4:17831] [[15336,1],6] tcp:send_handler SENDING TO [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 275
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 275 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 275 bytes from [[15336,0],3] for tag 2
[cn4:17831] [[15336,1],6] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 10 BYTES ON SOCKET 11
[cn4:17831] [[15336,1],6]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 159
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 159 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 159 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1237
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1237 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21419] [[15336,1],4] rml_send_buffer to peer [[15336,0],2] at tag 30
[cn3:21419] [[15336,1],4] grpcomm:bad barrier underway
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],7]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] message received 1237 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 10
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,1],7] (ORIGIN [[15336,1],7]) OF 10 BYTES FOR DEST [[15336,0],3] TAG 30
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21419] [[15336,1],4] rml_send_msg to peer [[15336,0],2] at tag 30
[cn3:21419] [[15336,1],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21419] [[15336,1],4] oob:base:send to target [[15336,0],2]
[cn3:21419] [[15336,1],4] oob:base:send known transport for peer [[15336,0],2]
[cn3:21419] [[15336,1],4] oob:tcp:send_nb to peer [[15336,0],2]:30
[cn3:21419] [[15336,1],4] tcp:send_nb to peer [[15336,0],2]
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:30
[cn3:21419] [[15336,1],4] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_nb: already connected to [[15336,0],2] - queueing for send
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_handler called to send to peer [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_handler SENDING TO [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 275
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 275 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 275 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] message received 10 bytes from [[15336,1],7] for tag 30
[cn4:17828] [[15336,0],3] COLLECTIVE RECVD FROM [[15336,1],7]
[cn3:21419] [[15336,1],4] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 10 BYTES ON SOCKET 11
[cn3:21419] [[15336,1],4]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],8]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 10
[cn4:17828] [[15336,0],3] WORKING COLLECTIVE 1
[cn4:17828] [[15336,0],3] PROGRESSING COLLECTIVE 1
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 0
[cn4:17828] [[15336,0],3] COLL 0 IS LOCALLY COMPLETE
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 2
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 32
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 642 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 32 BYTES FOR DEST [[15336,0],0] TAG 32
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 642 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 32 bytes from [[15336,0],2] for tag 32
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],2]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 1
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,1],8] (ORIGIN [[15336,1],8]) OF 10 BYTES FOR DEST [[15336,0],4] TAG 30
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 6
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] message received 10 bytes from [[15336,1],8] for tag 30
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] COLLECTIVE RECVD FROM [[15336,1],8]
[cn5:14171] [[15336,0],4] WORKING COLLECTIVE 1
[cn5:14171] [[15336,0],4] PROGRESSING COLLECTIVE 1
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 0
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 235 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] COLL 0 IS LOCALLY COMPLETE
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 1
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],5]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 10
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,1],5] (ORIGIN [[15336,1],5]) OF 10 BYTES FOR DEST [[15336,0],2] TAG 30
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] message received 10 bytes from [[15336,1],5] for tag 30
[cn3:21416] [[15336,0],2] COLLECTIVE RECVD FROM [[15336,1],5]
[cn3:21416] [[15336,0],2] WORKING COLLECTIVE 1
[cn3:21416] [[15336,0],2] PROGRESSING COLLECTIVE 1
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 2
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 0
[cn3:21416] [[15336,0],2] COLL 0 IS LOCALLY COMPLETE
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 1
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 2
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 847 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 235 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],2]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 10
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,1],2] (ORIGIN [[15336,1],2]) OF 10 BYTES FOR DEST [[15336,0],1] TAG 30
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2428 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] message received 10 bytes from [[15336,1],2] for tag 30
[cn2:01042] [[15336,0],1] COLLECTIVE RECVD FROM [[15336,1],2]
[cn2:01042] [[15336,0],1] WORKING COLLECTIVE 1
[cn2:01042] [[15336,0],1] PROGRESSING COLLECTIVE 1
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 0
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2428 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] COLL 0 IS LOCALLY COMPLETE
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 1
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 2
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],9]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 159 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 159 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 10
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2428 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,1],9] (ORIGIN [[15336,1],9]) OF 10 BYTES FOR DEST [[15336,0],4] TAG 30
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],6]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1237 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1298 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 10
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],4]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 10
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,1],6] (ORIGIN [[15336,1],6]) OF 10 BYTES FOR DEST [[15336,0],3] TAG 30
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,1],4] (ORIGIN [[15336,1],4]) OF 10 BYTES FOR DEST [[15336,0],2] TAG 30
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] message received 10 bytes from [[15336,1],9] for tag 30
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] COLLECTIVE RECVD FROM [[15336,1],9]
[cn5:14171] [[15336,0],4] WORKING COLLECTIVE 1
[cn5:14171] [[15336,0],4] PROGRESSING COLLECTIVE 1
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1237 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] message received 10 bytes from [[15336,1],4] for tag 30
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 0
[cn5:14171] [[15336,0],4] COLL 0 IS LOCALLY COMPLETE
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 1
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] message received 10 bytes from [[15336,1],6] for tag 30
[cn3:21416] [[15336,0],2] COLLECTIVE RECVD FROM [[15336,1],4]
[cn3:21416] [[15336,0],2] WORKING COLLECTIVE 1
[cn3:21416] [[15336,0],2] PROGRESSING COLLECTIVE 1
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 0
[cn3:21416] [[15336,0],2] COLL 0 IS LOCALLY COMPLETE
[cn5:14171] [[15336,0],4] COLLECTIVE 1 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],4] at tag 32
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 2
[cn4:17828] [[15336,0],3] COLLECTIVE RECVD FROM [[15336,1],6]
[cn4:17828] [[15336,0],3] WORKING COLLECTIVE 1
[cn4:17828] [[15336,0],3] PROGRESSING COLLECTIVE 1
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 0
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 1
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] COLLECTIVE 1 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] COLL 0 IS LOCALLY COMPLETE
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 1
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] COLLECTIVE 1 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],2] at tag 32
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 2
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],4] at tag 32
[cn5:14171] [[15336,0],4] rml_send_iovec_to_self at tag 32
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],3] at tag 32
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 2
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],2] at tag 32
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_iovec_to_self at tag 32
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_to_self callback executing for tag 32
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],3] at tag 32
[cn4:17828] [[15336,0],3] rml_send_iovec_to_self at tag 32
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] message received 32 bytes from [[15336,0],4] for tag 32
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],4]
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: WORKING COLLECTIVE 1
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_to_self callback executing for tag 32
[cn3:21416] [[15336,0],2] message received 32 bytes from [[15336,0],2] for tag 32
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_to_self callback executing for tag 32
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],2]
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: WORKING COLLECTIVE 1
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3] message received 32 bytes from [[15336,0],3] for tag 32
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],3]
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: WORKING COLLECTIVE 1
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],1]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],1] at tag 32
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 32
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:32
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 32
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:32
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],1] at tag 32
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:32
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],1]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],1]:32
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:32
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2428 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:32
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 275 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 343 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 275 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 32 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 32 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],1]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 32 BYTES ON SOCKET 10
[cn4:17828] [[15336,0],3]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 10
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,1],3] (ORIGIN [[15336,1],3]) OF 10 BYTES FOR DEST [[15336,0],1] TAG 30
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 32
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 32 BYTES FOR DEST [[15336,0],1] TAG 32
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] message received 10 bytes from [[15336,1],3] for tag 30
[cn2:01042] [[15336,0],1] COLLECTIVE RECVD FROM [[15336,1],3]
[cn2:01042] [[15336,0],1] WORKING COLLECTIVE 1
[cn2:01042] [[15336,0],1] PROGRESSING COLLECTIVE 1
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 0
[cn2:01042] [[15336,0],1] COLL 0 IS LOCALLY COMPLETE
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1611
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] COLLECTIVE 1 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],1] at tag 32
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 2
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] message received 32 bytes from [[15336,0],3] for tag 32
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],3]
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: WORKING COLLECTIVE 1
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],1] at tag 32
[cn2:01042] [[15336,0],1] rml_send_iovec_to_self at tag 32
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_to_self callback executing for tag 32
[cn2:01042] [[15336,0],1] message received 32 bytes from [[15336,0],1] for tag 32
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],1]
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: WORKING COLLECTIVE 1
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 4
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 32
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1611 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1611 bytes from [[15336,0],1] for tag 2
[cn2:01046] [[15336,1],3] rml_send_buffer to peer [[15336,0],1] at tag 30
[cn2:01046] [[15336,1],3] grpcomm:bad barrier underway
[cn2:01046] [[15336,1],3] rml_send_msg to peer [[15336,0],1] at tag 30
[cn2:01046] [[15336,1],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01046] [[15336,1],3] oob:base:send to target [[15336,0],1]
[cn2:01046] [[15336,1],3] oob:base:send known transport for peer [[15336,0],1]
[cn2:01046] [[15336,1],3] oob:tcp:send_nb to peer [[15336,0],1]:30
[cn2:01046] [[15336,1],3] tcp:send_nb to peer [[15336,0],1]
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:30
[cn2:01046] [[15336,1],3] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_handler called to send to peer [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_handler SENDING TO [[15336,0],1]
[cn2:01046] [[15336,1],3] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 10 BYTES ON SOCKET 11
[cn2:01046] [[15336,1],3]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 32
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 32 BYTES FOR DEST [[15336,0],0] TAG 32
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 32 bytes from [[15336,0],1] for tag 32
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],1]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 1
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 4
[cn1:14209] [[15336,0],0] grpcomm:bad:xcast sent to job [15336,1] tag 30
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 15
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 15
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 15
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 15
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:32
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 40 bytes from [[15336,0],0] for tag 15
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] grpcomm:xcast:recv: with 40 bytes
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:32
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],1]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],1] at tag 15
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],2]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1611 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],2] at tag 15
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],4]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],4] at tag 15
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 1
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 32 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],1] at tag 15
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],2] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],4] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 1
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 1
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],1]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],2]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],4]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],4]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],4]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 1
[cn1:14209] [[15336,0],0] message received 40 bytes from [[15336,0],0] for tag 1
[cn1:14209] [[15336,0],0] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn1:14209] [[15336,0],0] orted_cmd: received message_local_procs
[cn1:14209] [[15336,0],0] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,1],0] at tag 30
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,1],1] at tag 30
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],2] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],2]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],4] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,1],0] at tag 30
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,1],1] at tag 30
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,1],0]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,1],0]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,1],0]:30
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],0]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,1],1]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,1],1]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,1],1]:30
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],0]:30
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,1],0]) --> [[15336,1],0]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,1],0] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],0]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],1]:30
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,1],1]) --> [[15336,1],1]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,1],1] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],1]
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],2]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],2]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 40 BYTES ON SOCKET 17
[cn1:14209] [[15336,0],0]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,1],0]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,1],0]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,1],0] OF 10 BYTES ON SOCKET 20
[cn1:14209] [[15336,0],0]-[[15336,1],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,1],1]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,1],1]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,1],1] OF 10 BYTES ON SOCKET 0
[cn1:14209] [[15336,0],0]-[[15336,1],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],1]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],1]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 40 BYTES ON SOCKET 14
[cn1:14209] [[15336,0],0]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],4]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 40 BYTES ON SOCKET 15
[cn1:14209] [[15336,0],0]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14224] [[15336,1],0]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14224] [[15336,1],0]:tcp:recv:handler CONNECTED
[cn1:14224] [[15336,1],0]:tcp:recv:handler allocate new recv msg
[cn1:14224] [[15336,1],0]:tcp:recv:handler read hdr
[cn1:14224] [[15336,1],0]:tcp:recv:handler allocate data region of size 10
[cn1:14225] [[15336,1],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14225] [[15336,1],1]:tcp:recv:handler CONNECTED
[cn1:14225] [[15336,1],1]:tcp:recv:handler allocate new recv msg
[cn1:14225] [[15336,1],1]:tcp:recv:handler read hdr
[cn1:14225] [[15336,1],1]:tcp:recv:handler allocate data region of size 10
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 40
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 40
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 40
[cn1:14224] [[15336,1],0]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14224] [[15336,1],0]:tcp:recv:handler CONNECTED
[cn1:14224] [[15336,1],0] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 10 BYTES FOR DEST [[15336,1],0] TAG 30
[cn1:14224] [[15336,1],0] DELIVERING TO RML
[cn1:14224] [[15336,1],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14224] [[15336,1],0] message received 10 bytes from [[15336,0],0] for tag 30
[cn1:14224] [[15336,1],0] grpcomm:base:receive processing collective return for id 1 recvd from [[15336,0],0]
[cn1:14224] [[15336,1],0] CHECKING COLL id 1
[cn1:14225] [[15336,1],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14225] [[15336,1],1]:tcp:recv:handler CONNECTED
[cn1:14225] [[15336,1],1] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 10 BYTES FOR DEST [[15336,1],1] TAG 30
[cn1:14225] [[15336,1],1] DELIVERING TO RML
[cn1:14225] [[15336,1],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14225] [[15336,1],1] message received 10 bytes from [[15336,0],0] for tag 30
[cn1:14225] [[15336,1],1] grpcomm:base:receive processing collective return for id 1 recvd from [[15336,0],0]
[cn1:14225] [[15336,1],1] CHECKING COLL id 1
[cn1:14225] progress: set_event_flag setting to 2
[cn1:14224] progress: set_event_flag setting to 2
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 40 BYTES FOR DEST [[15336,0],2] TAG 15
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] message received 40 bytes from [[15336,0],0] for tag 15
[cn3:21416] [[15336,0],2] grpcomm:xcast:recv: with 40 bytes
[cn3:21416] [[15336,0],2] orte:daemon:send_relay - recipient list is empty!
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],2] at tag 1
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],2] at tag 1
[cn3:21416] [[15336,0],2] rml_send_iovec_to_self at tag 1
[cn3:21416] [[15336,0],2] rml_send_to_self callback executing for tag 1
[cn3:21416] [[15336,0],2] message received 40 bytes from [[15336,0],2] for tag 1
[cn3:21416] [[15336,0],2] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn3:21416] [[15336,0],2] orted_cmd: received message_local_procs
[cn3:21416] [[15336,0],2] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,1],4] at tag 30
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,1],5] at tag 30
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,1],4] at tag 30
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,1],5] at tag 30
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,1],4]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,1],4]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,1],4]:30
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,1],4]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],4]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,1],5]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,1],5]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,1],5]:30
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,1],5]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],5]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],4]:30
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,1],4]) --> [[15336,1],4]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,1],4] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],4]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],5]:30
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,1],5]) --> [[15336,1],5]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,1],5] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],5]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,1],5]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,1],5]
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 40 BYTES FOR DEST [[15336,0],1] TAG 15
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] message received 40 bytes from [[15336,0],0] for tag 15
[cn2:01042] [[15336,0],1] grpcomm:xcast:recv: with 40 bytes
[cn2:01042] [[15336,0],1] orte:daemon:send_relay sending relay msg to [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],3] at tag 15
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],1] at tag 1
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],3] at tag 15
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],1] at tag 1
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,1],5] OF 10 BYTES ON SOCKET 13
[cn2:01042] [[15336,0],1] rml_send_iovec_to_self at tag 1
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],3]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],3]:15
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn3:21416] [[15336,0],2]-[[15336,1],5] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,1],4]
[cn2:01042] [[15336,0],1] rml_send_to_self callback executing for tag 1
[cn2:01042] [[15336,0],1] message received 40 bytes from [[15336,0],1] for tag 1
[cn2:01042] [[15336,0],1] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn2:01042] [[15336,0],1] orted_cmd: received message_local_procs
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,1],4]
[cn2:01042] [[15336,0],1] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,1],2] at tag 30
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,1],4] OF 10 BYTES ON SOCKET 12
[cn3:21416] [[15336,0],2]-[[15336,1],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,1],3] at tag 30
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:15
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],3] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],3]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,1],2] at tag 30
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,1],3] at tag 30
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,1],2]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,1],2]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,1],2]:30
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,1],2]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],2]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,1],3]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 157 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 348
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 40
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,1],3]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,1],3]:30
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,1],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],2]:30
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,1],2]) --> [[15336,1],2]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,1],2] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],2]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],3]:30
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,1],3]) --> [[15336,1],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,1],3] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],3]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,1],2]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,1],2]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,1],2] OF 10 BYTES ON SOCKET 13
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 348 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]-[[15336,1],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,1],3]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,1],3]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,1],3] OF 10 BYTES ON SOCKET 14
[cn2:01042] [[15336,0],1]-[[15336,1],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],3]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 40 BYTES ON SOCKET 10
[cn2:01042] [[15336,0],1]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 348 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 348 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 40 BYTES FOR DEST [[15336,0],4] TAG 15
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] message received 40 bytes from [[15336,0],0] for tag 15
[cn5:14171] [[15336,0],4] grpcomm:xcast:recv: with 40 bytes
[cn5:14171] [[15336,0],4] orte:daemon:send_relay - recipient list is empty!
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],4] at tag 1
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],4] at tag 1
[cn5:14171] [[15336,0],4] rml_send_iovec_to_self at tag 1
[cn5:14171] [[15336,0],4] rml_send_to_self callback executing for tag 1
[cn5:14171] [[15336,0],4] message received 40 bytes from [[15336,0],4] for tag 1
[cn5:14171] [[15336,0],4] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn5:14171] [[15336,0],4] orted_cmd: received message_local_procs
[cn5:14171] [[15336,0],4] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,1],8] at tag 30
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,1],9] at tag 30
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,1],8] at tag 30
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,1],9] at tag 30
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,1],8]:30
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,1],8]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,1],9]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,1],9]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,1],9]:30
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,1],9]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],9]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],8]:30
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,1],8]) --> [[15336,1],8]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,1],8] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],8]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],9]:30
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,1],9]) --> [[15336,1],9]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,1],9] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],9]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,1],8]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,1],8]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,1],8] OF 10 BYTES ON SOCKET 12
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 273
[cn5:14171] [[15336,0],4]-[[15336,1],8] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,1],9]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,1],9]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,1],9] OF 10 BYTES ON SOCKET 13
[cn5:14171] [[15336,0],4]-[[15336,1],9] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 273 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 105 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 348 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14225] [[15336,1],1] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 113
[cn1:14225] [[15336,1],1] posting recv
[cn1:14225] [[15336,1],1] posting persistent recv on tag 113 for peer [[WILDCARD],WILDCARD]
[cn1:14224] [[15336,1],0] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 113
[cn1:14224] [[15336,1],0] posting recv
[cn1:14224] [[15336,1],0] posting persistent recv on tag 113 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],2] for tag 2
[cn3:21419] [[15336,1],4]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21419] [[15336,1],4]:tcp:recv:handler CONNECTED
[cn3:21419] [[15336,1],4]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 157
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 157 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 157 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 348
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 348 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] [[15336,1],4]:tcp:recv:handler read hdr
[cn3:21419] [[15336,1],4]:tcp:recv:handler allocate data region of size 10
[cn1:14209] [[15336,0],0] message received 348 bytes from [[15336,0],2] for tag 2
[cn3:21420] [[15336,1],5]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21420] [[15336,1],5]:tcp:recv:handler CONNECTED
[cn3:21420] [[15336,1],5]:tcp:recv:handler allocate new recv msg
[cn3:21420] [[15336,1],5]:tcp:recv:handler read hdr
[cn3:21420] [[15336,1],5]:tcp:recv:handler allocate data region of size 10
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 704
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 704 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 704 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 754
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 754 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] [[15336,1],4]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21419] [[15336,1],4]:tcp:recv:handler CONNECTED
[cn3:21419] [[15336,1],4] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 10 BYTES FOR DEST [[15336,1],4] TAG 30
[cn3:21419] [[15336,1],4] DELIVERING TO RML
[cn3:21419] [[15336,1],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] [[15336,1],4] message received 10 bytes from [[15336,0],2] for tag 30
[cn3:21419] [[15336,1],4] grpcomm:base:receive processing collective return for id 1 recvd from [[15336,0],2]
[cn3:21419] [[15336,1],4] CHECKING COLL id 1
[cn1:14209] [[15336,0],0] message received 754 bytes from [[15336,0],2] for tag 2
[cn3:21420] [[15336,1],5]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21420] [[15336,1],5]:tcp:recv:handler CONNECTED
[cn3:21420] [[15336,1],5] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 10 BYTES FOR DEST [[15336,1],5] TAG 30
[cn3:21420] [[15336,1],5] DELIVERING TO RML
[cn3:21420] [[15336,1],5] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] [[15336,1],5] message received 10 bytes from [[15336,0],2] for tag 30
[cn3:21420] [[15336,1],5] grpcomm:base:receive processing collective return for id 1 recvd from [[15336,0],2]
[cn3:21420] [[15336,1],5] CHECKING COLL id 1
[cn3:21420] progress: set_event_flag setting to 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 348 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 348 bytes from [[15336,0],1] for tag 2
[cn2:01046] [[15336,1],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01046] [[15336,1],3]:tcp:recv:handler CONNECTED
[cn2:01046] [[15336,1],3]:tcp:recv:handler allocate new recv msg
[cn2:01046] [[15336,1],3]:tcp:recv:handler read hdr
[cn2:01046] [[15336,1],3]:tcp:recv:handler allocate data region of size 10
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 348
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 348 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 348 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 704
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 704 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 273 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] [[15336,1],2]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01045] [[15336,1],2]:tcp:recv:handler CONNECTED
[cn2:01045] [[15336,1],2]:tcp:recv:handler allocate new recv msg
[cn2:01045] [[15336,1],2]:tcp:recv:handler read hdr
[cn2:01045] [[15336,1],2]:tcp:recv:handler allocate data region of size 10
[cn1:14209] [[15336,0],0] message received 704 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 273 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 754
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 754 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 105
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 105 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01046] [[15336,1],3]:tcp:recv:handler CONNECTED
[cn2:01046] [[15336,1],3] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 10 BYTES FOR DEST [[15336,1],3] TAG 30
[cn2:01046] [[15336,1],3] DELIVERING TO RML
[cn2:01046] [[15336,1],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] message received 10 bytes from [[15336,0],1] for tag 30
[cn2:01046] [[15336,1],3] grpcomm:base:receive processing collective return for id 1 recvd from [[15336,0],1]
[cn2:01046] [[15336,1],3] CHECKING COLL id 1
[cn5:14175] [[15336,1],9]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14175] [[15336,1],9]:tcp:recv:handler CONNECTED
[cn5:14175] [[15336,1],9]:tcp:recv:handler allocate new recv msg
[cn5:14175] [[15336,1],9]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0] message received 754 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 105 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 348
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 348 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] [[15336,1],2]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01045] [[15336,1],2]:tcp:recv:handler CONNECTED
[cn2:01045] [[15336,1],2] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 10 BYTES FOR DEST [[15336,1],2] TAG 30
[cn2:01045] [[15336,1],2] DELIVERING TO RML
[cn2:01045] [[15336,1],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] [[15336,1],2] message received 10 bytes from [[15336,0],1] for tag 30
[cn2:01045] [[15336,1],2] grpcomm:base:receive processing collective return for id 1 recvd from [[15336,0],1]
[cn2:01045] [[15336,1],2] CHECKING COLL id 1
[cn2:01045] progress: set_event_flag setting to 2
[cn5:14175] [[15336,1],9]:tcp:recv:handler allocate data region of size 10
[cn3:21419] progress: set_event_flag setting to 2
[cn1:14209] [[15336,0],0] message received 348 bytes from [[15336,0],4] for tag 2
[cn5:14174] [[15336,1],8]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14174] [[15336,1],8]:tcp:recv:handler CONNECTED
[cn5:14174] [[15336,1],8]:tcp:recv:handler allocate new recv msg
[cn5:14174] [[15336,1],8]:tcp:recv:handler read hdr
[cn5:14174] [[15336,1],8]:tcp:recv:handler allocate data region of size 10
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 40 BYTES FOR DEST [[15336,0],3] TAG 15
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] message received 40 bytes from [[15336,0],1] for tag 15
[cn4:17828] [[15336,0],3] grpcomm:xcast:recv: with 40 bytes
[cn4:17828] [[15336,0],3] orte:daemon:send_relay - recipient list is empty!
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],3] at tag 1
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],3] at tag 1
[cn4:17828] [[15336,0],3] rml_send_iovec_to_self at tag 1
[cn4:17828] [[15336,0],3] rml_send_to_self callback executing for tag 1
[cn4:17828] [[15336,0],3] message received 40 bytes from [[15336,0],3] for tag 1
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 704 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] orted_cmd: received message_local_procs
[cn4:17828] [[15336,0],3] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,1],6] at tag 30
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,1],7] at tag 30
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,1],6] at tag 30
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 754 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,1],7] at tag 30
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,1],6]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,1],6]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,1],6]:30
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,1],6]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],6]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,1],7]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 704 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 754 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 348
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,1],7]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,1],7]:30
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,1],7]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],7]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],6]:30
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,1],6]) --> [[15336,1],6]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,1],6] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],6]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],7]:30
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,1],7]) --> [[15336,1],7]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,1],7] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],7]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,1],6]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,1],6]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,1],6] OF 10 BYTES ON SOCKET 13
[cn4:17828] [[15336,0],3]-[[15336,1],6] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,1],7]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,1],7]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,1],7] OF 10 BYTES ON SOCKET 14
[cn4:17828] [[15336,0],3]-[[15336,1],7] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 348 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 348 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 348 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 348
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 348 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] [[15336,1],6]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17831] [[15336,1],6]:tcp:recv:handler CONNECTED
[cn4:17831] [[15336,1],6]:tcp:recv:handler allocate new recv msg
[cn4:17831] [[15336,1],6]:tcp:recv:handler read hdr
[cn4:17831] [[15336,1],6]:tcp:recv:handler allocate data region of size 10
[cn1:14209] [[15336,0],0] message received 348 bytes from [[15336,0],3] for tag 2
[cn4:17832] [[15336,1],7]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17832] [[15336,1],7]:tcp:recv:handler CONNECTED
[cn4:17832] [[15336,1],7]:tcp:recv:handler allocate new recv msg
[cn4:17832] [[15336,1],7]:tcp:recv:handler read hdr
[cn4:17832] [[15336,1],7]:tcp:recv:handler allocate data region of size 10
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 348 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 704
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 704 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 704 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 754
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 754 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14175] [[15336,1],9]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14175] [[15336,1],9]:tcp:recv:handler CONNECTED
[cn5:14175] [[15336,1],9] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 10 BYTES FOR DEST [[15336,1],9] TAG 30
[cn5:14175] [[15336,1],9] DELIVERING TO RML
[cn5:14175] [[15336,1],9] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14175] [[15336,1],9] message received 10 bytes from [[15336,0],4] for tag 30
[cn5:14175] [[15336,1],9] grpcomm:base:receive processing collective return for id 1 recvd from [[15336,0],4]
[cn5:14175] [[15336,1],9] CHECKING COLL id 1
[cn1:14209] [[15336,0],0] message received 754 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14174] [[15336,1],8]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14174] [[15336,1],8]:tcp:recv:handler CONNECTED
[cn5:14174] [[15336,1],8] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 10 BYTES FOR DEST [[15336,1],8] TAG 30
[cn5:14174] [[15336,1],8] DELIVERING TO RML
[cn5:14174] [[15336,1],8] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14174] [[15336,1],8] message received 10 bytes from [[15336,0],4] for tag 30
[cn5:14174] [[15336,1],8] grpcomm:base:receive processing collective return for id 1 recvd from [[15336,0],4]
[cn5:14174] [[15336,1],8] CHECKING COLL id 1
[cn5:14174] progress: set_event_flag setting to 2
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],4] for tag 2
[cn5:14175] progress: set_event_flag setting to 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 704 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 114 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 754 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 114 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 245 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 245 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 161 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 161 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 114
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 114 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 114 bytes from [[15336,0],2] for tag 2
[cn3:21419] [[15336,1],4] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 113
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 245
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 245 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 245 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] [[15336,1],5] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 113
[cn3:21420] [[15336,1],5] posting recv
[cn3:21420] [[15336,1],5] posting persistent recv on tag 113 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 161
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 161 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn1:14209] [[15336,0],0] message received 161 bytes from [[15336,0],2] for tag 2
[cn3:21419] [[15336,1],4] posting recv
[cn3:21419] [[15336,1],4] posting persistent recv on tag 113 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0xf09f40 (lid 7, ep index 4)
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],2] for tag 2
endpoint 0x85ece0 (lid 7, ep index 4)
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 114
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 114 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 245 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 223 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 114 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 114 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 245
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 245 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14175] [[15336,1],9] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 113
[cn1:14209] [[15336,0],0] message received 245 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 223
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 223 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14174] [[15336,1],8] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 113
[cn5:14174] [[15336,1],8] posting recv
[cn5:14174] [[15336,1],8] posting persistent recv on tag 113 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0] message received 223 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0x1168440 (lid 8, ep index 0)
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 161 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 161
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 161 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 161 bytes from [[15336,0],4] for tag 2
[cn5:14175] [[15336,1],9] posting recv
[cn5:14175] [[15336,1],9] posting persistent recv on tag 113 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
endpoint 0xbf08a0 (lid 8, ep index 1)
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 593
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 593 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 593 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 593 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xf09f40
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xf2b240
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xf2b240 with type 100[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] progress: event_users_increment setting count to 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],1] for tag 2

[cn2:01046] progress: set_event_flag setting to 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 114
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 114 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 114 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 245
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 245 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 113
[cn1:14209] [[15336,0],0] message received 245 bytes from [[15336,0],1] for tag 2
[cn2:01045] [[15336,1],2] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 113
[cn2:01045] [[15336,1],2] posting recv
[cn2:01045] [[15336,1],2] posting persistent recv on tag 113 for peer [[WILDCARD],WILDCARD]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 161
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 161 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 609
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 609 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn1:14209] [[15336,0],0] message received 161 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 609 bytes from [[15336,0],2] for tag 2
[cn2:01046] [[15336,1],3] posting recv
[cn2:01046] [[15336,1],3] posting persistent recv on tag 113 for peer [[WILDCARD],WILDCARD]
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870912, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x85ece0, rem_ep = 0xf09f40, src qpn = 33170, length = 120, local buffer # = 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0xa69640 (lid 4, ep index 2)
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 215
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 215 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
endpoint 0x1adcec0 (lid 4, ep index 2)
[cn1:14209] [[15336,0],0] message received 215 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xa69640
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 215 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 409 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 610 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 597 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 609 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1660 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 341 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1347 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 378 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 357 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 51 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 888 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 204 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 51 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 379 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 379 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 378 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 51 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 597
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 597 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 597 bytes from [[15336,0],4] for tag 2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870912, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x14df2e0, rem_ep = 0xbf08a0, src qpn = 27544, length = 120, local buffer # = 0
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xc150a0 on ep 0x14df2e0
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x14f6120 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870912, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xec28e0, rem_ep = 0x1168440, src qpn = 27543, length = 120, local buffer # = 0
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn2:01046] progress: event_users_increment setting count to 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0x1168440
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x11dbd00
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x11dbd00 with type 100
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 409
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 409 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 409 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xad3f80
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xad3f80 with type 100
[cn5:14174] progress: event_users_increment setting count to 2
sending request for endpoint 0x85ece0[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 610
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 610 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 610 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870912, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x1adcec0, rem_ep = 0xa69640, src qpn = 37963, length = 120, local buffer # = 0
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] 
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x14f62e0 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xbf08a0[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
activating timeout for message 0x880100sending request for endpoint 0x1adcec0
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] 
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xc150a0[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

created message 0x880100 with type 100activating timeout for message 0x1b00300[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] 

[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x14fb2e0 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xc150a0 with type 100[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xf2b240 on ep 0x85ece0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 341
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 341 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 341 bytes from [[15336,0],2] for tag 2
created message 0x1b00300 with type 100
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870912, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1347
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1347 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 1347 bytes from [[15336,0],2] for tag 2

[cn5:14175] progress: event_users_increment setting count to 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xf09f40, rem_ep = 0x85ece0, src qpn = 33171, length = 120, local buffer # = 0
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x880100 on ep 0xf09f40
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870913, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xf2b240 from slid 0x0007 qp 0x00008193
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xad3f80 on ep 0x1adcec0
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870912, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xc150a0 from slid 0x0003 qp 0x0000913e
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf253e0 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] progress: event_users_increment setting count to 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1660
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1660 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 1660 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 888
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 888 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] progress: event_users_increment setting count to 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870912, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xa69640, rem_ep = 0x1adcec0, src qpn = 37962, length = 120, local buffer # = 0
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1b00300 on ep 0xa69640
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870913, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xad3f80 from slid 0x0004 qp 0x0000944a
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x14fb920 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 888 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870913, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x880100 from slid 0x0007 qp 0x00008192
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xa621c0 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x87a080 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],1] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xad3320 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 378
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 378 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 378 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xad3940 to RTS
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x87a240 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 51
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 51 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 51 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 357
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 357 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 357 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xad89e0 to RTS
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 51
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 51 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 51 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf255a0 to RTS
successfully set RTS
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870913, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1b00300 from slid 0x0004 qp 0x0000944b
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 204
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 204 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 204 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf25be0 to RTS
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1afa220 to RTS
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf2ac00 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 379
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 379 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 379 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1afa3e0 to RTS
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 379
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 379 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 379 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1aff500 to RTS
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1affb40 to RTS

[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 378
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 378 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x87a880 to RTS
[cn1:14209] [[15336,0],0] message received 378 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x87fae0 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 51
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 51 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 51 bytes from [[15336,0],2] for tag 2
successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x11dbd00 on ep 0xec28e0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 486
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 486 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 259 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 486 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 486 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 259
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 259 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870912, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] [cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xedac20 to RTS
[cn1:14209] [[15336,0],0] message received 259 bytes from [[15336,0],4] for tag 2
got ack for message 0x11dbd00 from slid 0x0003 qp 0x0000913d
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xedae60 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xedfde0 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xee0420 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 583 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 583
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 583 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 583 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xbcfa40
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xbcfa40 with type 101
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 609
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 609 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 609 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870914, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x1adcec0, rem_ep = 0xa69640, src qpn = 37963, length = 88, local buffer # = 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],1] for tag 2
activating timeout for message 0x1bf6a00[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 609 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
created message 0x1bf6a00 with type 101[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 585
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xbcfa40 on ep 0x1adcec0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1485
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1485 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1485 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870914, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xa69640, rem_ep = 0x1adcec0, src qpn = 37962, length = 88, local buffer # = 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1bf6a00 on ep 0xa69640
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870915, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xbcfa40 from slid 0x0004 qp 0x0000944a
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn2:01046] progress: event_users_decrement setting count to 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 574
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 574 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 585 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 574 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 585 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 608
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 608 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xa69640.
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x1021bc0
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x1021bc0 with type 101
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 608 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870915, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1bf6a00 from slid 0x0004 qp 0x0000944b
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870914, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x85ece0, rem_ep = 0xf09f40, src qpn = 33170, length = 88, local buffer # = 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 74
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 74 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn1:14209] [[15336,0],0] message received 74 bytes from [[15336,0],1] for tag 2
finishing connection for endpoint 0x1adcec0.[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 585 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 608 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1485 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 574 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 74 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1854 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 204 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 53 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 223 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 53 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
activating timeout for message 0x976500[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
created message 0x976500 with type 101[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1854
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1854 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1021bc0 on ep 0x85ece0
[cn1:14209] [[15336,0],0] message received 1854 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870914, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xf09f40, rem_ep = 0x85ece0, src qpn = 33171, length = 88, local buffer # = 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x976500 on ep 0xf09f40
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870915, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1021bc0 from slid 0x0007 qp 0x00008193
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xf09f40.
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 204
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 204 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 204 bytes from [[15336,0],2] for tag 2

[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] progress: event_users_decrement setting count to 1
[cn3:21419] progress: event_users_decrement setting count to 1
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870915, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x976500 from slid 0x0007 qp 0x00008192
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0xa68640 (lid 4, ep index 1)
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 53
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 53 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 53 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
finishing connection for endpoint 0x85ece0.exiting message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 223
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 223 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 223 bytes from [[15336,0],1] for tag 2

[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0x1a8b720 (lid 4, ep index 0)
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn3:21420] progress: event_users_decrement setting count to 1
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0x14df2e0
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x15f2400
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x15f2400 with type 100
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x15f2660
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x15f2660 with type 101
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 450
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 450 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] progress: event_users_increment setting count to 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870913, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x15f2400 from slid 0x0008 qp 0x00006b98
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870914, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x15f2660 from slid 0x0008 qp 0x00006b98
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0] message received 450 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],2] for tag 2
sending request for endpoint 0x1a8b720
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x1bf9ec0
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x1bf9ec0 with type 100
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0xf0b2e0 (lid 7, ep index 5)
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 53
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 53 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 53 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
exiting message thread
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0x8613e0 (lid 7, ep index 6)
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1573
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1573 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 450 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 341 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 556 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 298 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 486 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 259 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 204 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 341 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 298 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 204 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1573 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 341
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 341 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 556
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 556 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 1573 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 341 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 556 bytes from [[15336,0],2] for tag 2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870915, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x1485a40, rem_ep = 0xa68640, src qpn = 37963, length = 120, local buffer # = 3
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xbd31a0 on ep 0x1485a40
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x15f25e0 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x15f4880 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x15f4ec0 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x15f5500 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870913, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xebb3e0, rem_ep = 0x1a8b720, src qpn = 37962, length = 120, local buffer # = 1
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1bf9ec0 on ep 0xebb3e0
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870913, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xbf08a0, rem_ep = 0x14df2e0, src qpn = 37182, length = 120, local buffer # = 1
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x15f2400 on ep 0xbf08a0
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870914, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xbf08a0, rem_ep = 0x14df2e0, src qpn = 37182, length = 88, local buffer # = 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x15f2660 on ep 0xbf08a0
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870912, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xf0b2e0
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x10250c0
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0x1485a40
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x161afe0
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x161afe0 with type 100
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x161b1e0
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x161b1e0 with type 101
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 298
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 298 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 298 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],2] for tag 2
sending request for endpoint 0xa68640[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xe76d20, rem_ep = 0xf0b2e0, src qpn = 33170, length = 120, local buffer # = 0
created message 0x10250c0 with type 100
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870916, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x161afe0 from slid 0x0004 qp 0x0000944b
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870917, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x161b1e0 from slid 0x0004 qp 0x0000944b
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2

running message thread[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x10250c0 on ep 0xe76d20
[cn3:21419] progress: event_users_increment setting count to 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 204
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 204 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 204 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] 
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870916, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x10250c0 from slid 0x0005 qp 0x00007e47
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
activating timeout for message 0xbd31a0[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 486
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 486 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

transitioning QP 0xc0f2c0 to RTStransitioning QP 0xe8e560 to RTSsending request for endpoint 0x8613e0[cn1:14209] [[15336,0],0] message received 486 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 341
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 341 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870916, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] 


[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 259
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 259 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 341 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 259 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 298
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 298 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870912, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] got ack for message 0x1bf9ec0 from slid 0x0003 qp 0x0000913d
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 298 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xdfad40, rem_ep = 0x8613e0, src qpn = 33171, length = 120, local buffer # = 0
activating timeout for message 0x9799c0[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] successfully set RTS[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
created message 0xbd31a0 with type 100
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] 
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] successfully set RTS[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x9799c0 on ep 0xdfad40
created message 0x9799c0 with type 100[cn2:01046] progress: event_users_increment setting count to 2
transitioning QP 0xc0f480 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 204
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 204 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 204 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1573
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1573 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870916, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xbd31a0 from slid 0x0003 qp 0x0000913e
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message

[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread

[cn1:14209] [[15336,0],0] message received 1573 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870917, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xa68640, rem_ep = 0x1485a40, src qpn = 37182, length = 120, local buffer # = 5
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x161afe0 on ep 0xa68640
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870918, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xa68640, rem_ep = 0x1485a40, src qpn = 37182, length = 88, local buffer # = 6
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x161b1e0 on ep 0xa68640
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] 
[cn3:21420] progress: event_users_increment setting count to 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],1] for tag 2
successfully set RTS[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xe11560 to RTS
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870916, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x9799c0 from slid 0x0005 qp 0x00007e48
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
running message thread
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xe8c460 to RTS

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xc0fac0 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0xbcfc40 to RTS
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xe0f460 to RTS
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],3] for tag 2

[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 51
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 51 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 51 bytes from [[15336,0],3] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTSsuccessfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
successfully set RTS
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xe10780 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],3] for tag 2

[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xc14a60 to RTS[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xe0fb20 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0xbd1ee0 to RTS
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2

[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTStransitioning QP 0xe8d780 to RTS
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
successfully set RTS
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2

successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] 
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],1] for tag 2
transitioning QP 0xbd2520 to RTS[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 51 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1573 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870918, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x1485a40, rem_ep = 0xa68640, src qpn = 37963, length = 88, local buffer # = 6
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xbf85a0 on ep 0x1485a40
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0x1485a40.
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14225] progress: event_users_decrement setting count to 0

[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xe8cb20 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
successfully set RTSsuccessfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2


[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],1] for tag 2
transitioning QP 0xbd2b60 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
successfully set RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
activating timeout for message 0xbf85a0[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],1] for tag 2
created message 0xbf85a0 with type 101[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],1] for tag 2
finishing connection for endpoint 0xa68640.[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870919, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xbf85a0 from slid 0x0003 qp 0x0000913e
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2

[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
[cn2:01046] progress: event_users_decrement setting count to 1
activating timeout for message 0xd0bb00[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
exiting message thread
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870919, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x14df2e0, rem_ep = 0xbf08a0, src qpn = 27544, length = 88, local buffer # = 7
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xd0bb00 on ep 0x14df2e0
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2

[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
created message 0xd0bb00 with type 101[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0x1486d20 (lid 3, ep index 0)
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870915, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xd0bb00 from slid 0x0003 qp 0x0000913e
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],4] for tag 2
finishing connection for endpoint 0xbf08a0.[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],4] for tag 2
[cn5:14175] progress: event_users_decrement setting count to 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
exiting message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1569 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 215 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 583 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1398 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 704 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 704 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1743 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 601 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1113 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 51 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 345 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 232 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 608 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 607 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 114 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 114 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 161 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 204 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 608 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 161 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 215 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 156 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 953 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 609 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 608 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1080 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1569
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1569 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1569 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0x1486d20
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x161e700
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x161e700 with type 100
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0x14df2e0.
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870920, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x161e700 from slid 0x0003 qp 0x0000913d
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14225] progress: event_users_increment setting count to 1
[cn1:14225] progress: event_users_decrement setting count to 0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870914, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xeba060, rem_ep = 0x1486d20, src qpn = 37182, length = 120, local buffer # = 2
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x161e700 on ep 0xeba060
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870917, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x8613e0, rem_ep = 0xdfad40, src qpn = 32328, length = 120, local buffer # = 5
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xf09580 on ep 0x8613e0
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870918, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x8613e0, rem_ep = 0xdfad40, src qpn = 32328, length = 88, local buffer # = 6
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xf097e0 on ep 0x8613e0
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
sending request for endpoint 0xdfad40[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

running message thread[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] 
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],3] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] activating timeout for message 0xf09580[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 215
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 215 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

transitioning QP 0x976480 to RTS[cn1:14209] [[15336,0],0] message received 215 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xf09580 with type 100

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 583
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 583 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 583 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xf097e0
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xf097e0 with type 101
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1398
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1398 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1398 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870913, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xf09580 from slid 0x0007 qp 0x00008193
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870914, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xf097e0 from slid 0x0007 qp 0x00008193
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 704
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 704 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 704 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17831] [[15336,1],6]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17831] [[15336,1],6]:tcp:recv:handler CONNECTED
[cn4:17831] [[15336,1],6] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 10 BYTES FOR DEST [[15336,1],6] TAG 30
[cn4:17831] [[15336,1],6] DELIVERING TO RML
[cn4:17831] [[15336,1],6] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] [[15336,1],6] message received 10 bytes from [[15336,0],3] for tag 30
[cn4:17831] [[15336,1],6] grpcomm:base:receive processing collective return for id 1 recvd from [[15336,0],3]
[cn4:17831] [[15336,1],6] CHECKING COLL id 1
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 704
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 704 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 704 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17832] [[15336,1],7]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17832] [[15336,1],7]:tcp:recv:handler CONNECTED
[cn4:17832] [[15336,1],7] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 10 BYTES FOR DEST [[15336,1],7] TAG 30
[cn4:17832] [[15336,1],7] DELIVERING TO RML
[cn4:17832] [[15336,1],7] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17832] [[15336,1],7] message received 10 bytes from [[15336,0],3] for tag 30
[cn4:17832] [[15336,1],7] grpcomm:base:receive processing collective return for id 1 recvd from [[15336,0],3]
[cn4:17832] [[15336,1],7] CHECKING COLL id 1
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 400 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1743
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1743 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17832] progress: set_event_flag setting to 2
transitioning QP 0x978700 to RTS[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 156 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 1743 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870917, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xf0b2e0, rem_ep = 0xe76d20, src qpn = 32327, length = 120, local buffer # = 5
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xf86d80 on ep 0xf0b2e0
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870918, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xf0b2e0, rem_ep = 0xe76d20, src qpn = 32327, length = 88, local buffer # = 6
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xf86f00 on ep 0xf0b2e0
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 601
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 378 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 601 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 601 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1113
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1113 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 378 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
sending request for endpoint 0xe76d20
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xf86d80
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xf86d80 with type 100
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1021b40 to RTS
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 1113 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
activating timeout for message 0xf86f00
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xf86f00 with type 101
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870913, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xf86d80 from slid 0x0007 qp 0x00008192
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message

[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 378 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 583 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] progress: set_event_flag setting to 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 608 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870914, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xf86f00 from slid 0x0007 qp 0x00008192
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 608
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 608 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 608 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
successfully set RTS[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870915, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xe76d20, rem_ep = 0xf0b2e0, src qpn = 33170, length = 88, local buffer # = 3
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 114
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 114 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x104a7a0 on ep 0xe76d20
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x978d40 to RTS
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 114 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 114
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 114 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17831] [[15336,1],6] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 113
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 114 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 161
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 161 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17832] [[15336,1],7] rml_recv_buffer_nb for peer [[WILDCARD],WILDCARD] tag 113
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x979380 to RTS
[cn1:14209] [[15336,0],0] message received 161 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],2] for tag 2
[cn4:17831] [[15336,1],6] posting recv
[cn4:17831] [[15336,1],6] posting persistent recv on tag 113 for peer [[WILDCARD],WILDCARD]
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0xe79380 (lid 5, ep index 6)

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 608
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 608 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 608 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870915, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xdfad40, rem_ep = 0x8613e0, src qpn = 33171, length = 88, local buffer # = 3
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 2027 BYTES ON SOCKET 9
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 161
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 161 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 161 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17832] [[15336,1],7] posting recv
[cn4:17832] [[15336,1],7] posting persistent recv on tag 113 for peer [[WILDCARD],WILDCARD]
transitioning QP 0x1023e20 to RTS[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x99efe0 on ep 0xdfad40

[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0xdfc380 (lid 5, ep index 6)
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 215
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 215 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 51
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 51 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 215 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 51 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xdfc380
successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xec28e0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xfd7160
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xfd7160 with type 100
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xfd73c0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xfd73c0 with type 101
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 156
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 156 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 156 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 953
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 953 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 345
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 345 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1024440 to RTS
[cn4:17832] progress: event_users_increment setting count to 2
[cn4:17832] progress: event_users_decrement setting count to 1
[cn1:14209] [[15336,0],0] message received 953 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 345 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xf15400
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xf15400 with type 100
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xdfad40.
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870915, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xfd7160 from slid 0x0008 qp 0x00006b97
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870916, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xfd73c0 from slid 0x0008 qp 0x00006b97
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 609
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 609 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 232
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 232 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 609 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 232 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 609
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870916, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xe79380, rem_ep = 0xdfc380, src qpn = 32328, length = 120, local buffer # = 4
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
transitioning QP 0x1024a80 to RTS
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 609 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0] message received 609 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1168
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1168 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870913, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x1168440, rem_ep = 0xec28e0, src qpn = 37181, length = 120, local buffer # = 1sending request for endpoint 0xe79380activating timeout for message 0x99efe0[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 1168 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xfd7160 on ep 0x1168440
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870914, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x1168440, rem_ep = 0xec28e0, src qpn = 37181, length = 88, local buffer # = 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xfd73c0 on ep 0x1168440
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2


[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 609 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 379
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 379 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 607
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 607 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x11d5da0 to RTS
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 379 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1168 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 607 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 53 BYTES ON SOCKET 9
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x11d5f60 to RTS
activating timeout for message 0xf926e0[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x104a7a0
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x104a7a0 with type 101
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xf0b2e0.
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 379 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 379
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 379 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 379 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 204
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 204 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn3:21419] progress: event_users_decrement setting count to 1
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 379 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 204 bytes from [[15336,0],2] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x11db080 to RTS
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],2] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x11db6c0 to RTS
created message 0xf926e0 with type 100[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870919, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x104a7a0 from slid 0x0005 qp 0x00007e47
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],2] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS

created message 0x99efe0 with type 101[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 608
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 608 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 608 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870916, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xdfc380, rem_ep = 0xe79380, src qpn = 32327, length = 120, local buffer # = 4
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xf15400 on ep 0xe79380
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1080
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1080 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1080 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],2] for tag 2
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xfd75c0 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS

[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xf926e0 on ep 0xdfc380
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870917, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xf15400 from slid 0x0005 qp 0x00007e47
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
finishing connection for endpoint 0x8613e0.[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf11ea0 to RTS

[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn3:21420] progress: event_users_decrement setting count to 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf14140 to RTS
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870919, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x99efe0 from slid 0x0005 qp 0x00007e48
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
exiting message thread[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf14780 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS

[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870917, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xebdae0, rem_ep = 0xf051a0, src qpn = 33170, length = 120, local buffer # = 5
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x104ddc0 on ep 0xebdae0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xfd9840 to RTS
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 492 BYTES ON SOCKET 9
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf14dc0 to RTS
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870917, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xf926e0 from slid 0x0005 qp 0x00007e48
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 514 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 400
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 400 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 594 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 410 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 400 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xe76d20.
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 156
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 156 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 156 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 378
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 378 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn4:17831] progress: event_users_increment setting count to 2
[cn4:17831] progress: event_users_decrement setting count to 1
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
endpoint 0xf051a0 (lid 7, ep index 0)[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 378 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf8f1a0 to RTS
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS

[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf91420 to RTS
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 378
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 378 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 216 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 378 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf91a60 to RTS
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
endpoint 0x85b260 (lid 7, ep index 1)
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 492
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 492 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 410
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 410 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 492 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 410 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 216
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 216 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870917, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] [cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xf051a0
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x104ddc0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 514
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 514 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 378
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 378 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 216 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 514 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 378 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x104ddc0 with type 100
received message. type: 100, lcl_ep = 0x1ae0300, rem_ep = 0xe74340, src qpn = 32327, length = 120, local buffer # = 5
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xfbab20 on ep 0x1ae0300
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xf920a0 to RTS
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21419] progress: event_users_increment setting count to 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 583
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 583 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 583 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
transitioning QP 0x1bfa0a0 to RTS[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xfb73c0
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xfb73c0 with type 101
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870920, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x104ddc0 from slid 0x0003 qp 0x0000913d
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 608
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 608 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 608 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2

[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870918, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xdfc380, rem_ep = 0xe79380, src qpn = 32327, length = 88, local buffer # = 6
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870921, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x14da180, rem_ep = 0x85b260, src qpn = 33171, length = 120, local buffer # = 9
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x9a2600 on ep 0x14da180
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
activating timeout for message 0xf3a4e0[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
successfully set RTS[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 341 BYTES ON SOCKET 9

[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 663 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
sending request for endpoint 0x85b260[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 341
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 341 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]

[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
created message 0xf3a4e0 with type 101
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 341 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870920, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]

[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 663
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 663 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 663 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xaba320, rem_ep = 0xdf88a0, src qpn = 32328, length = 120, local buffer # = 8
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xf3dc20 on ep 0xaba320
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xfb73c0 on ep 0xdfc380
activating timeout for message 0x9a2600[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 2027
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 2027 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 2027 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xbf8860 to RTS
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870918, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xe79380, rem_ep = 0xdfc380, src qpn = 32328, length = 88, local buffer # = 6
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xf3a4e0 on ep 0xe79380
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870919, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xfb73c0 from slid 0x0005 qp 0x00007e48
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xe79380.
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread

[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn4:17831] progress: event_users_decrement setting count to 0
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870919, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xf3a4e0 from slid 0x0005 qp 0x00007e47
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
created message 0x9a2600 with type 100[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] 
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 64 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1570 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1147 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 486 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 942 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
finishing connection for endpoint 0xdfc380.[cn3:21420] progress: event_users_increment setting count to 2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xfd9f20 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xfda540 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xebb3e0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x10000a0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x10000a0 with type 100
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x10002a0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x10002a0 with type 101
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870918, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x10000a0 from slid 0x0004 qp 0x0000944a
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870919, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x10002a0 from slid 0x0004 qp 0x0000944a
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x10004a0 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1bfc320 to RTS

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17832] progress: event_users_decrement setting count to 0
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870920, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x9a2600 from slid 0x0003 qp 0x0000913e
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1575 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1bfc960 to RTS
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0xe74340 (lid 5, ep index 2)
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 53
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 53 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 53 bytes from [[15336,0],3] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
exiting message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1002720 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
transitioning QP 0xbfaae0 to RTS
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0xdf88a0 (lid 5, ep index 3)
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 594
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 594 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17831] progress: event_users_increment setting count to 1
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 594 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xe74340
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xfbab20
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xfbab20 with type 100
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 64
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 64 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 64 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x161ea80 to RTS
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
transitioning QP 0x1bfcfa0 to RTS
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870920, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xfbab20 from slid 0x0004 qp 0x0000944a
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 610 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 543 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xbf8660 to RTS
sending request for endpoint 0xdf88a0[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 231 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1002d60 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS

[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xbfb5e0 to RTS
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
activating timeout for message 0xf3dc20[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] 
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 378 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
sending request for endpoint 0x1ae0300[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x10033a0 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1147
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1147 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 1147 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 486
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 486 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
created message 0xf3dc20 with type 100[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xaba320
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xc25c20
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xc25c20 with type 100
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xc25e20
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xc25e20 with type 101
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0] message received 486 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870921, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] [cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2

[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]

[cn4:17832] progress: event_users_increment setting count to 1
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 942
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 942 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 942 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
got ack for message 0xc25c20 from slid 0x0005 qp 0x00007e48
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870922, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xc25e20 from slid 0x0005 qp 0x00007e48
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870920, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xf3dc20 from slid 0x0004 qp 0x0000944b
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1575
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1575 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1570
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1570 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1575 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 1570 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870918, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x1a8b720, rem_ep = 0xebb3e0, src qpn = 37181, length = 120, local buffer # = 6
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x10000a0 on ep 0x1a8b720
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870919, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x1a8b720, rem_ep = 0xebb3e0, src qpn = 37181, length = 88, local buffer # = 7
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x10002a0 on ep 0x1a8b720
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870921, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xdf88a0, rem_ep = 0xaba320, src qpn = 37963, length = 120, local buffer # = 9
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xc25c20 on ep 0xdf88a0
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870922, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xdf88a0, rem_ep = 0xaba320, src qpn = 37963, length = 88, local buffer # = 10
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xc25e20 on ep 0xdf88a0
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
activating timeout for message 0x1c22bc0[cn4:17828] [[15336,0],3] routing direct to the HNP
running message thread[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2


[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0xf3a8a0 to RTScreated message 0x1c22bc0 with type 100[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870920, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xec28e0, rem_ep = 0x1168440, src qpn = 27543, length = 88, local buffer # = 8
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x12d2c00 on ep 0xec28e0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xeba060
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x102dba0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x102dba0 with type 100
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x102dda0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x102dda0 with type 101
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870922, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x1486d20, rem_ep = 0xeba060, src qpn = 37181, length = 120, local buffer # = 10
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x102dba0 on ep 0x1486d20
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870923, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x1486d20, rem_ep = 0xeba060, src qpn = 37181, length = 88, local buffer # = 11
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x102dda0 on ep 0x1486d20
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]

[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]

[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870921, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x102dba0 from slid 0x0003 qp 0x0000913e
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870922, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x102dda0 from slid 0x0003 qp 0x0000913e
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 610 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870920, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1c22bc0 from slid 0x0005 qp 0x00007e47
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
activating timeout for message 0x12d2c00successfully set RTS[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 610
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 409 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 610 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
activating timeout for message 0x1c22dc0

[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x102dfa0 to RTS
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 610 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 543
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 543 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870921, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xe74340, rem_ep = 0x1ae0300, src qpn = 37962, length = 120, local buffer # = 9
[cn4:17828] [[15336,0],3] routing direct to the HNP

[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 543 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 231
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1486 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 231 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1c22bc0 on ep 0xe74340
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x12d2c00 with type 101[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 231 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 592 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

transitioning QP 0xfb75c0 to RTS
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
created message 0x1c22dc0 with type 101[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 74
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 74 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]

[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xfb9860 to RTS
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 74 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 378
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 378 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] finishing connection for endpoint 0x1168440.[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1030240 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 378 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
transitioning QP 0x1bf6980 to RTS[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870915, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x12d2c00 from slid 0x0003 qp 0x0000913d
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xfb9ea0 to RTS
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1486
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1486 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x161e700 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS


[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xfba4e0 to RTS
[cn1:14209] [[15336,0],0] message received 1486 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870921, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x1ae0300, rem_ep = 0xe74340, src qpn = 32327, length = 88, local buffer # = 9
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xfe4ae0 on ep 0x1ae0300
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870922, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1c22dc0 from slid 0x0005 qp 0x00007e47
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn5:14174] progress: event_users_decrement setting count to 1
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1030880 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870923, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xebb3e0, rem_ep = 0x1a8b720, src qpn = 37962, length = 88, local buffer # = 11
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1c4d0e0 on ep 0xebb3e0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
successfully set RTSexiting message threadtransitioning QP 0xf3a6e0 to RTS[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 610
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 610 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 610 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]


[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870922, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xe74340, rem_ep = 0x1ae0300, src qpn = 37962, length = 88, local buffer # = 10
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]

[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1030f60 to RTS
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 409
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 409 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 409 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
transitioning QP 0x1bf8c00 to RTS[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xfe4ae0
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xfe4ae0 with type 101
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9

[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17831] progress: event_users_decrement setting count to 0
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 592
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 592 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 592 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1c22dc0 on ep 0xe74340
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xe74340.
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
successfully set RTS[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870923, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xfe4ae0 from slid 0x0004 qp 0x0000944a
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP

[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
successfully set RTS[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
transitioning QP 0x1bf9240 to RTS[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]

[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],3] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] transitioning QP 0xf3cfa0 to RTS[cn1:14224] progress: event_users_decrement setting count to 0
[cn1:14224] progress: event_users_decrement setting count to -1
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xebdae0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x1058c60
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x1058c60 with type 100
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x1058e60
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x1058e60 with type 101
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xec28e0.
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xebb3e0.
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2

[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 492
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 492 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 492 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1284
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] successfully set RTS[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870921, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] [cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0x1bf9880 to RTS
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1284 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 1284 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] transitioning QP 0xf3d5e0 to RTSreceived message. type: 100, lcl_ep = 0xf051a0, rem_ep = 0xebdae0, src qpn = 37181, length = 120, local buffer # = 9
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1058c60 on ep 0xf051a0
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870922, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xf051a0, rem_ep = 0xebdae0, src qpn = 37181, length = 88, local buffer # = 10
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1058e60 on ep 0xf051a0
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
successfully set RTS
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2

[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] transitioning QP 0x104aa60 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 492
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 492 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 492 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870923, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] 

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 882
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 882 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] progress: event_users_decrement setting count to 0
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 882 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
received message. type: 101, lcl_ep = 0xaba320, rem_ep = 0xdf88a0, src qpn = 32328, length = 88, local buffer # = 11
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xf67be0 on ep 0xaba320
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xaba320.
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
activating timeout for message 0xf67be0successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
activating timeout for message 0x1c4d0e0

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],3] for tag 2

[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xf67be0 with type 101[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
created message 0x1c4d0e0 with type 101
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2

[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] transitioning QP 0x104a7a0 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 156
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 156 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 156 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 598
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 598 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] progress: event_users_decrement setting count to 1
[cn2:01045] progress: event_users_decrement setting count to 0
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870923, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xf67be0 from slid 0x0004 qp 0x0000944b
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message

[cn1:14209] [[15336,0],0] message received 598 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0x1a8b720.
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0x1ae0300.
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
finishing connection for endpoint 0xdf88a0.[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870923, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1c4d0e0 from slid 0x0003 qp 0x0000913d
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
successfully set RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],3] for tag 2
[cn4:17832] progress: event_users_decrement setting count to 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] 
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
exiting message thread[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2

transitioning QP 0x104d140 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
successfully set RTS[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1621180 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS

[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0x104d780 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x16217c0 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],2] for tag 2
activating timeout for message 0x1077de0[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0x14da180
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x164be20
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x164be20 with type 100
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x164c020
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x164c020 with type 101
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870924, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x164be20 from slid 0x0007 qp 0x00008193
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870925, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x164c020 from slid 0x0007 qp 0x00008193
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x161b1a0 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
created message 0x1077de0 with type 101[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x161d440 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 802
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 802 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 802 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870921, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x85b260, rem_ep = 0x14da180, src qpn = 37182, length = 120, local buffer # = 9
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x164be20 on ep 0x85b260
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x161da80 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 976
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 976 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 976 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870922, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x85b260, rem_ep = 0x14da180, src qpn = 37182, length = 88, local buffer # = 10
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x164c020 on ep 0x85b260
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x99f2a0 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x161e0c0 to RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 226
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 226 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 226 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xf051a0.
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 204
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 204 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 204 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] progress: event_users_decrement setting count to 1
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0x99efe0 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 74 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 492 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 882 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 156 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 598 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 492 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1284 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 802 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 976 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 226 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 204 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
successfully set RTS[cn1:14224] progress: event_users_decrement setting count to -2
[cn1:14224] progress: event_users_decrement setting count to -3
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870924, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1058c60 from slid 0x0007 qp 0x00008192
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870925, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1058e60 from slid 0x0007 qp 0x00008192
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870926, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xebdae0, rem_ep = 0xf051a0, src qpn = 33170, length = 88, local buffer # = 14
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1077de0 on ep 0xebdae0
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870927, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xeba060, rem_ep = 0x1486d20, src qpn = 37182, length = 88, local buffer # = 15
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x16738a0 on ep 0xeba060
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xebdae0.
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xeba060.
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14225] progress: event_users_decrement setting count to -1
[cn1:14225] progress: event_users_decrement setting count to -2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x16738a0
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x16738a0 with type 101
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0x1486d20.
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870926, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x16738a0 from slid 0x0003 qp 0x0000913d
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870927, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x14da180, rem_ep = 0x85b260, src qpn = 33171, length = 88, local buffer # = 15
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x9cc5a0 on ep 0x14da180
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0x14da180.
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
transitioning QP 0x9a1980 to RTS[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
endpoint 0x11bfbc0 (lid 8, ep index 8)
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870923, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1077de0 from slid 0x0003 qp 0x0000913d
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
transitioning QP 0x9a1fc0 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
successfully set RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
activating timeout for message 0x9cc5a0[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
created message 0x9cc5a0 with type 101[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870923, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x9cc5a0 from slid 0x0003 qp 0x0000913e
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
finishing connection for endpoint 0x85b260.[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn3:21420] progress: event_users_decrement setting count to 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
exiting message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0x14d8b40 (lid 3, ep index 3)
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0xbf8e20 (lid 8, ep index 8)
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
sending request for endpoint 0x11bfbc0[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 411
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 411 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 411 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x12df1a0
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x12df1a0 with type 100
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 610
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 610 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14174] progress: event_users_increment setting count to 2
[cn1:14209] [[15336,0],0] message received 610 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870916, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xbf8e20, rem_ep = 0x11bfbc0, src qpn = 27543, length = 120, local buffer # = 4
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
sending request for endpoint 0xbf8e20[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
activating timeout for message 0xd180a0[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
created message 0xd180a0 with type 100[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x12df1a0 on ep 0xbf8e20
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1487
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1487 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1487 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 204
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 204 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870916, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x11bfbc0, rem_ep = 0xbf8e20, src qpn = 27544, length = 120, local buffer # = 4
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xd180a0 on ep 0x11bfbc0
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870917, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x12df1a0 from slid 0x0008 qp 0x00006b98
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0] message received 204 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x12dbc20 to RTS
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 379
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 379 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 379 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x12ddf00 to RTS
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 379
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 379 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 379 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x12de520 to RTS
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x12deb60 to RTS
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 223 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 546
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 546 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 546 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870917, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xd180a0 from slid 0x0008 qp 0x00006b97
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 372
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 372 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 372 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14175] progress: event_users_increment setting count to 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xd14b20 to RTS
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 378
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 378 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 378 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 411 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xd16e00 to RTS
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 378
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 378 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 610 BYTES ON SOCKET 9
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xd17420 to RTS
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 378 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xd17a60 to RTS
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 583
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 583 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 583 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xd3d6e0
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xd3d6e0 with type 101
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 609
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 609 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 609 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870918, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x11bfbc0, rem_ep = 0xbf8e20, src qpn = 27544, length = 88, local buffer # = 6
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
activating timeout for message 0x1304820[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]

[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
created message 0x1304820 with type 101[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2

[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xd3d6e0 on ep 0x11bfbc0
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1485
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1485 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 1485 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 803
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870918, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xbf8e20, rem_ep = 0x11bfbc0, src qpn = 27543, length = 88, local buffer # = 6
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1304820 on ep 0xbf8e20
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870919, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xd3d6e0 from slid 0x0008 qp 0x00006b97
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14175] progress: event_users_decrement setting count to 1
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 574
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1487 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 574 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 574 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 204 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xbf8e20.
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870919, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1304820 from slid 0x0008 qp 0x00006b98
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 411 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 223
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 223 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 74
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 74 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 803 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 223 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 74 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 803 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 379 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0x1adef60 (lid 4, ep index 4)
finishing connection for endpoint 0x11bfbc0.[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870924, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x85c5e0, rem_ep = 0x1adef60, src qpn = 37962, length = 120, local buffer # = 12
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1c68980 on ep 0x85c5e0
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 411
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 411 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 803 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
endpoint 0xabb880 (lid 4, ep index 7)

[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 411 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 379 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0x1adef60
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x1c68980
[cn5:14174] progress: event_users_decrement setting count to 1
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 216
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 216 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 216 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01045] progress: event_users_increment setting count to 1
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] running message thread[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 216 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x1c68980 with type 100
exiting message thread
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 546 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870924, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1c68980 from slid 0x0007 qp 0x00008193
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 372 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
transitioning QP 0x9d7100 to RTS[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]

[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 378 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 802
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
sending request for endpoint 0xabb880[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 802 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 378 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 802 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870920, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x11b9ce0, rem_ep = 0xabb880, src qpn = 37963, length = 120, local buffer # = 8
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xc33c80 on ep 0x11b9ce0
successfully set RTS[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 583 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] 
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
activating timeout for message 0xc33c80[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
running message thread[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]

[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 609 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] 
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
created message 0xc33c80 with type 100[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2

[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01046] progress: event_users_increment setting count to 1
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0x9d94a0 to RTS[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870924, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xc33c80 from slid 0x0008 qp 0x00006b97
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
transitioning QP 0x13127e0 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2

[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP


[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1485 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 574 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
successfully set RTS[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 74 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
transitioning QP 0x1314ac0 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
transitioning QP 0x9d9b40 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]

[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 802 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
successfully set RTS[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send

[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
successfully set RTS[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]

[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
transitioning QP 0x1315140 to RTS[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
transitioning QP 0x9da160 to RTS[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]

[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
successfully set RTS[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]

[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
successfully set RTS[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]

[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],4] for tag 2
transitioning QP 0x1315760 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0x14d8b40
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x1681620
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x1681620 with type 100
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870928, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1681620 from slid 0x0007 qp 0x00008192
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14225] progress: event_users_increment setting count to -1
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 492
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 492 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 492 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870929, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x14d8b40, rem_ep = 0xf06540, src qpn = 33170, length = 120, local buffer # = 17
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x10b6bc0 on ep 0x14d8b40
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870925, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] successfully set RTS[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1287
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1287 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 1287 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
sending request for endpoint 0x85c5e0received message. type: 100, lcl_ep = 0x1adef60, rem_ep = 0x85c5e0, src qpn = 33171, length = 120, local buffer # = 13
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x9fd440 on ep 0x1adef60
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870926, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x1adef60, rem_ep = 0x85c5e0, src qpn = 33171, length = 88, local buffer # = 14
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x9fd640 on ep 0x1adef60
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 977
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 977 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 977 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1c653e0 to RTS
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870924, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xf06540, rem_ep = 0x14d8b40, src qpn = 37182, length = 120, local buffer # = 12
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1681620 on ep 0xf06540
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 379
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 379 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 379 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 602
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 602 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1c67680 to RTS
sending request for endpoint 0x11b9ce0[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 379
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 379 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 602 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 379 bytes from [[15336,0],1] for tag 2

[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x9fd440
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x9fd440 with type 100
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x9fd640[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1c67d20 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870930, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x14d8b40, rem_ep = 0xf06540, src qpn = 33170, length = 88, local buffer # = 18
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x10b6dc0 on ep 0x14d8b40
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 390
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 390 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 390 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] 
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x9fd640 with type 101
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 211
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 211 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1398
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1398 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 211 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 1398 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1c68340 to RTS
activating timeout for message 0x133b3a0[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870925, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x9fd440 from slid 0x0004 qp 0x0000944a
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870926, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x9fd640 from slid 0x0004 qp 0x0000944a
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS

[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
transitioning QP 0x1090a40 to RTS[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
activating timeout for message 0x1c8b700created message 0x133b3a0 with type 100
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 977
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 977 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 977 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870925, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xabb880, rem_ep = 0x11b9ce0, src qpn = 27543, length = 120, local buffer # = 13
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x133b3a0 on ep 0xabb880
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread

[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xc305c0 to RTS
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 378
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 378 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
activating timeout for message 0x133b5a0[cn1:14209] [[15336,0],0] message received 378 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xc32980 to RTS
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870921, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x133b3a0 from slid 0x0004 qp 0x0000944b
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xc33020 to RTS

transitioning QP 0x1092cc0 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] 
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 210
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 210 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 210 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 198
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 198 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0xc33640 to RTS
created message 0x133b5a0 with type 101[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 198 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS

successfully set RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] 
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 340
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 340 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 340 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 300
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 300 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870926, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008exiting message thread[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 300 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1487
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1487 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xabb880, rem_ep = 0x11b9ce0, src qpn = 27543, length = 88, local buffer # = 14

transitioning QP 0x1093360 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1487 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870922, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x11b9ce0, rem_ep = 0xabb880, src qpn = 37963, length = 88, local buffer # = 10
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xc56980 on ep 0x11b9ce0
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870923, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x133b5a0 from slid 0x0004 qp 0x0000944b
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3:21420] progress: event_users_decrement setting count to 0
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 797
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 797 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1346
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1346 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 797 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 1346 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0xc56980
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0xc56980 with type 101
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x133b5a0 on ep 0xabb880
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xabb880.
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870927, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x85c5e0, rem_ep = 0x1adef60, src qpn = 37962, length = 88, local buffer # = 15
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1c8b700 on ep 0x85c5e0
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0x85c5e0.
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2:01046] progress: event_users_decrement setting count to 0
running message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 888
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 888 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 888 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870927, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xc56980 from slid 0x0008 qp 0x00006b97
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message

[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 74
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 74 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
created message 0x1c8b700 with type 101[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 74 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
finishing connection for endpoint 0x11b9ce0.

[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]

[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 492 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870927, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1c8b700 from slid 0x0007 qp 0x00008193
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn5:14174] progress: event_users_decrement setting count to 0
transitioning QP 0x1093980 to RTS[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 74
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 74 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 74 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
finishing connection for endpoint 0x1adef60.[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] 
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1287 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 977 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] 
exiting message thread[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01045] progress: event_users_decrement setting count to 0

successfully set RTS[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] 
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
exiting message thread[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
sending request for endpoint 0xf06540[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]

[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 379 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 602 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
activating timeout for message 0x10b6bc0[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 379 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
created message 0x10b6bc0 with type 100[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 390 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2

[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870925, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x10b6bc0 from slid 0x0003 qp 0x0000913e
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
activating timeout for message 0x10b6dc0[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1398 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
created message 0x10b6dc0 with type 101[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send

[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 211 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870926, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x10b6dc0 from slid 0x0003 qp 0x0000913e
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
running message thread[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
exiting message thread[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]

[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]

[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
transitioning QP 0x167df60 to RTS[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]

[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
successfully set RTS[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9

[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 977 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
transitioning QP 0x1680320 to RTS[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2

[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
successfully set RTS[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
transitioning QP 0x16809c0 to RTS[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]

[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 378 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
successfully set RTS[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
transitioning QP 0x1680fe0 to RTS[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2

[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 210 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
successfully set RTS[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 198 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 340 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 300 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1487 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 797 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
activating timeout for message 0x16a2260[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]

[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1346 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
created message 0x16a2260 with type 101[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send

[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 888 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
finishing connection for endpoint 0x14d8b40.[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send

[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 74 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870931, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x16a2260 from slid 0x0007 qp 0x00008192
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14225] progress: event_users_decrement setting count to -2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 802
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 802 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
exiting message thread
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 802 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870927, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xf06540, rem_ep = 0x14d8b40, src qpn = 37182, length = 88, local buffer # = 15
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x16a2260 on ep 0xf06540
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 74 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 802 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 609 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 396 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 609
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 609 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 609 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 396
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 396 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870920, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xbf4140, rem_ep = 0xf0ed20, src qpn = 33170, length = 120, local buffer # = 8
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 396 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x10ba160 on ep 0xbf4140
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
running message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] 
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],2] for tag 2
transitioning QP 0xd3efe0 to RTS[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

finishing connection for endpoint 0xf06540.[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] 
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
successfully set RTS[cn3:21419] progress: event_users_decrement setting count to 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] exiting message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0xd413a0 to RTS
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2

[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
endpoint 0xf0ed20 (lid 7, ep index 8)[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2

successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
sending request for endpoint 0xf0ed20[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536


[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],2] for tag 2
transitioning QP 0xd41a40 to RTSactivating timeout for message 0x10ba160[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2


[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
created message 0x10ba160 with type 100[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
successfully set RTS
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2

[cn3:21419] progress: event_users_increment setting count to 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870928, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x10ba160 from slid 0x0008 qp 0x00006b98
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
transitioning QP 0xd42060 to RTS[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
successfully set RTS[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
sending request for endpoint 0xbf4140[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
activating timeout for message 0xd67ca0[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
created message 0xd67ca0 with type 100[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 801
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
activating timeout for message 0xd67ea0[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870921, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xd67ca0 from slid 0x0007 qp 0x00008192
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 801 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 801 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 801 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870929, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xf0ed20, rem_ep = 0xbf4140, src qpn = 27544, length = 120, local buffer # = 17
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xd67ca0 on ep 0xf0ed20
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
created message 0xd67ea0 with type 101[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
running message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2

[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2

[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
exiting message thread[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 609 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 609
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 609 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

transitioning QP 0x10b6fc0 to RTS[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 609 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870930, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xf0ed20, rem_ep = 0xbf4140, src qpn = 27544, length = 88, local buffer # = 18
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]

[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
successfully set RTS[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
transitioning QP 0x10b6c00 to RTS[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
successfully set RTS[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]

[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
transitioning QP 0x10b6de0 to RTS[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
transitioning QP 0x10b9b20 to RTS[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2

[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
successfully set RTS[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
activating timeout for message 0x10df720[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send

[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
created message 0x10df720 with type 101[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1485
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1485 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 221
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 221 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1485 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 221 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 221 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870922, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xbf4140, rem_ep = 0xf0ed20, src qpn = 33170, length = 88, local buffer # = 10
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x10df720 on ep 0xbf4140
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870923, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xd67ea0 from slid 0x0007 qp 0x00008192
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xd67ea0 on ep 0xf0ed20
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1485 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870931, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x10df720 from slid 0x0008 qp 0x00006b98
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
finishing connection for endpoint 0xf0ed20.[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
running message thread[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]

[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
finishing connection for endpoint 0xbf4140.[cn3:21419] progress: event_users_decrement setting count to 0
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14175] progress: event_users_decrement setting count to 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
exiting message threadexiting message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2


[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] sending rkey 1342482745, rdma_start.lval 7fd46cf36074, pval 0x7fd46cf36074, ival 1827889268 type 1 and sizeof(rdma_hdr) 16

[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
prior to NTOH received  rkey 1342482745, rdma_start.lval 7fd46cf36074, pval 0x7fd46cf36074, ival 1827889268[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]

[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] received  rkey 1342482745, rdma_start.lval 7fd46cf36074, pval 0x7fd46cf36074, ival 1827889268
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] prior to NTOH received  rkey 1207999820, rdma_start.lval 7f36bf859074, pval 0x7f36bf859074, ival 3213201524
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
received  rkey 1207999820, rdma_start.lval 7f36bf859074, pval 0x7f36bf859074, ival 3213201524[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],3] for tag 2
sending rkey 1610649492, rdma_start.lval 7f1cc1fdc074, pval 0x7f1cc1fdc074, ival 3254632564 type 1 and sizeof(rdma_hdr) 16
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],1] for tag 2
sending rkey 1207999820, rdma_start.lval 7f36bf859074, pval 0x7f36bf859074, ival 3213201524 type 1 and sizeof(rdma_hdr) 16
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] prior to NTOH received  rkey 1208000076, rdma_start.lval 7fe89afbf074, pval 0x7fe89afbf074, ival 2600202356
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] received  rkey 1208000076, rdma_start.lval 7fe89afbf074, pval 0x7fe89afbf074, ival 2600202356
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],1] for tag 2
prior to NTOH received  rkey 1610649492, rdma_start.lval 7f1cc1fdc074, pval 0x7f1cc1fdc074, ival 3254632564[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],1] for tag 2
received  rkey 1610649492, rdma_start.lval 7f1cc1fdc074, pval 0x7f1cc1fdc074, ival 3254632564[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],1] for tag 2
sending rkey 1208000076, rdma_start.lval 7fe89afbf074, pval 0x7fe89afbf074, ival 2600202356 type 1 and sizeof(rdma_hdr) 16
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],3] for tag 2
sending rkey 1610649748, rdma_start.lval 7f589ae1e074, pval 0x7f589ae1e074, ival 2598494324 type 1 and sizeof(rdma_hdr) 16
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] sending rkey 1342483001, rdma_start.lval 7f953f757074, pval 0x7f953f757074, ival 1064661108 type 1 and sizeof(rdma_hdr) 16

[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] prior to NTOH received  rkey 1342483001, rdma_start.lval 7f953f757074, pval 0x7f953f757074, ival 1064661108
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] received  rkey 1342483001, rdma_start.lval 7f953f757074, pval 0x7f953f757074, ival 1064661108
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] prior to NTOH received  rkey 536913311, rdma_start.lval 7f0dabf44074, pval 0x7f0dabf44074, ival 2884911220
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 152 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 152
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 152 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 152 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 310
sending rkey 536913311, rdma_start.lval 7f0dabf44074, pval 0x7f0dabf44074, ival 2884911220 type 1 and sizeof(rdma_hdr) 16
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 310 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 310 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 504 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 504 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 309 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 152 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 502 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 136 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 310 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 310 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] sending rkey 1610650260, rdma_start.lval 7f5891de8074, pval 0x7f5891de8074, ival 2447278196 type 1 and sizeof(rdma_hdr) 16


[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],2] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 309
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 309 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 309 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 152
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 152 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
prior to NTOH received  rkey 1610649748, rdma_start.lval 7f589ae1e074, pval 0x7f589ae1e074, ival 2598494324sending rkey 1610650004, rdma_start.lval 7f1cb8fa6074, pval 0x7f1cb8fa6074, ival 3103416436 type 1 and sizeof(rdma_hdr) 16
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] sending rkey 536913823, rdma_start.lval 7f8d85a12074, pval 0x7f8d85a12074, ival 2241929332 type 1 and sizeof(rdma_hdr) 16

[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 152 bytes from [[15336,0],2] for tag 2


sending rkey 536913567, rdma_start.lval 7f0da2f0e074, pval 0x7f0da2f0e074, ival 2733695092 type 1 and sizeof(rdma_hdr) 16
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 502
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 502 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] 
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 502 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],1] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] prior to NTOH received  rkey 536913823, rdma_start.lval 7f8d85a12074, pval 0x7f8d85a12074, ival 2241929332
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] received  rkey 536913823, rdma_start.lval 7f8d85a12074, pval 0x7f8d85a12074, ival 2241929332
received  rkey 1610649748, rdma_start.lval 7f589ae1e074, pval 0x7f589ae1e074, ival 2598494324[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 504
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 504 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 504 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 136
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 136 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] prior to NTOH received  rkey 1610650260, rdma_start.lval 7f5891de8074, pval 0x7f5891de8074, ival 2447278196
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] received  rkey 1610650260, rdma_start.lval 7f5891de8074, pval 0x7f5891de8074, ival 2447278196
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 310
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 310 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 136 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 310 bytes from [[15336,0],1] for tag 2
prior to NTOH received  rkey 1610650004, rdma_start.lval 7f1cb8fa6074, pval 0x7f1cb8fa6074, ival 3103416436prior to NTOH received  rkey 536913567, rdma_start.lval 7f0da2f0e074, pval 0x7f0da2f0e074, ival 2733695092[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] sending rkey 1208000588, rdma_start.lval 7fe891f89074, pval 0x7fe891f89074, ival 2448986228 type 1 and sizeof(rdma_hdr) 16

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] 

[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
sending rkey 1208000332, rdma_start.lval 7f36b6823074, pval 0x7f36b6823074, ival 3061985396 type 1 and sizeof(rdma_hdr) 16
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 122
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 122 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 122 bytes from [[15336,0],2] for tag 2

received  rkey 1610650004, rdma_start.lval 7f1cb8fa6074, pval 0x7f1cb8fa6074, ival 3103416436received  rkey 536913567, rdma_start.lval 7f0da2f0e074, pval 0x7f0da2f0e074, ival 2733695092[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2


[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 122 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] received  rkey 536913311, rdma_start.lval 7f0dabf44074, pval 0x7f0dabf44074, ival 2884911220
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] sending rkey 1342483257, rdma_start.lval 7f95366f0074, pval 0x7f95366f0074, ival 913244276 type 1 and sizeof(rdma_hdr) 16

[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 504
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 504 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn1:14209] [[15336,0],0] message received 504 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] prior to NTOH received  rkey 1208000588, rdma_start.lval 7fe891f89074, pval 0x7fe891f89074, ival 2448986228
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] received  rkey 1208000588, rdma_start.lval 7fe891f89074, pval 0x7fe891f89074, ival 2448986228
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],1] for tag 2
prior to NTOH received  rkey 1208000332, rdma_start.lval 7f36b6823074, pval 0x7f36b6823074, ival 3061985396[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],1] for tag 2
received  rkey 1208000332, rdma_start.lval 7f36b6823074, pval 0x7f36b6823074, ival 3061985396[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 136
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 136 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 136 bytes from [[15336,0],1] for tag 2
prior to NTOH received  rkey 1342483257, rdma_start.lval 7f95366f0074, pval 0x7f95366f0074, ival 913244276[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 136 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 122 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],3] for tag 2
sending rkey 1610650516, rdma_start.lval 7f1cb8f41074, pval 0x7f1cb8f41074, ival 3103002740 type 1 and sizeof(rdma_hdr) 16
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 152 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] prior to NTOH received  rkey 1610650516, rdma_start.lval 7f1cb8f41074, pval 0x7f1cb8f41074, ival 3103002740[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
prior to NTOH received  rkey 1208000844, rdma_start.lval 7f36b67be074, pval 0x7f36b67be074, ival 3061571700
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],3] for tag 2
received  rkey 1610650516, rdma_start.lval 7f1cb8f41074, pval 0x7f1cb8f41074, ival 3103002740[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
received  rkey 1208000844, rdma_start.lval 7f36b67be074, pval 0x7f36b67be074, ival 3061571700
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 136 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 122 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 152
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 152 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 152 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] sending rkey 536914079, rdma_start.lval 7f0da2ea9074, pval 0x7f0da2ea9074, ival 2733281396 type 1 and sizeof(rdma_hdr) 16
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 136
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 136 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 136 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
prior to NTOH received  rkey 536914079, rdma_start.lval 7f0da2ea9074, pval 0x7f0da2ea9074, ival 2733281396
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 122
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 122 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn1:14209] [[15336,0],0] message received 122 bytes from [[15336,0],3] for tag 2
received  rkey 536914079, rdma_start.lval 7f0da2ea9074, pval 0x7f0da2ea9074, ival 2733281396[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] sending rkey 1342483513, rdma_start.lval 7f95366bc074, pval 0x7f95366bc074, ival 913031284 type 1 and sizeof(rdma_hdr) 16

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 136
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 136 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 136 bytes from [[15336,0],2] for tag 2
sending rkey 1342483769, rdma_start.lval 7f9536688074, pval 0x7f9536688074, ival 912818292 type 1 and sizeof(rdma_hdr) 16
prior to NTOH received  rkey 1342483513, rdma_start.lval 7f95366bc074, pval 0x7f95366bc074, ival 913031284[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 136 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 122
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 122 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 122 bytes from [[15336,0],2] for tag 2
received  rkey 1342483513, rdma_start.lval 7f95366bc074, pval 0x7f95366bc074, ival 913031284[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 122 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14225] progress: event_users_decrement setting count to -3
[cn1:14225] progress: progress_set_yield_when_idle to false
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 136
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 136 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 136 bytes from [[15336,0],4] for tag 2
prior to NTOH received  rkey 1342483769, rdma_start.lval 7f9536688074, pval 0x7f9536688074, ival 912818292[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 154 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 136 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 154 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 57 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 154 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 57 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 57 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 90 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 154 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 57 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 90 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 90 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 122 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 57 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 57 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 154
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2


[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 122
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 122 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 154 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn1:14209] [[15336,0],0] message received 122 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 154 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 57
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 57 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 154
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 154 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
received  rkey 1342483257, rdma_start.lval 7f95366f0074, pval 0x7f95366f0074, ival 913244276[cn4:17831] progress: event_users_decrement setting count to -1
[cn4:17831] progress: progress_set_yield_when_idle to false
[cn3:21419] progress: event_users_decrement setting count to -1
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 57 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 154 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 154
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 154 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 57
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 57 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 57
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 57 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn4:17832] progress: event_users_decrement setting count to -1
[cn3:21420] progress: event_users_decrement setting count to -1
[cn3:21420] progress: progress_set_yield_when_idle to false
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 154 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 57 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 57 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 90
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 90 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 90
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 90 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn5:14175] progress: event_users_decrement setting count to -1
[cn5:14175] progress: progress_set_yield_when_idle to false
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 90 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 90 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 122
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 122 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 57
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 57 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 57
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 57 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
sending rkey 1208000844, rdma_start.lval 7f36b67be074, pval 0x7f36b67be074, ival 3061571700 type 1 and sizeof(rdma_hdr) 16
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn4:17832] progress: progress_set_yield_when_idle to false
[cn3:21419] progress: progress_set_yield_when_idle to false
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 122 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 57 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 57 bytes from [[15336,0],2] for tag 2

received  rkey 1342483769, rdma_start.lval 7f9536688074, pval 0x7f9536688074, ival 912818292[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 154
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 154 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 154 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn2:01046] progress: event_users_decrement setting count to -1
[cn2:01046] progress: progress_set_yield_when_idle to false

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 57
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 57 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] progress: event_users_decrement setting count to -1
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0] message received 57 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 90
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 90 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
sending rkey 2013302201, rdma_start.lval 7fe491b0f074, pval 0x7fe491b0f074, ival 2444292212 type 1 and sizeof(rdma_hdr) 16
[cn1:14209] [[15336,0],0] message received 90 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 57
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 57 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] progress: progress_set_yield_when_idle to false

[cn1:14209] [[15336,0],0] message received 57 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],4] for tag 2
[cn5:14174] progress: event_users_decrement setting count to -1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 90
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 90 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 90 bytes from [[15336,0],4] for tag 2
[cn5:14174] progress: progress_set_yield_when_idle to false
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 57 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 90 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 57 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 57
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 57 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 57 bytes from [[15336,0],4] for tag 2
[cn1:14224] progress: event_users_decrement setting count to -4
[cn1:14224] progress: progress_set_yield_when_idle to false
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] endpoint 0x14898a0 (lid 3, ep index 1)
prior to NTOH received  rkey 2013302201, rdma_start.lval 7fe491b0f074, pval 0x7fe491b0f074, ival 2444292212
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] received  rkey 2013302201, rdma_start.lval 7fe491b0f074, pval 0x7fe491b0f074, ival 2444292212
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0x14898a0
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x1707a40
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x1707a40 with type 100
[cn1:14225] progress: event_users_increment setting count to -2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870932, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1707a40 from slid 0x0004 qp 0x0000944a
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 805 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 805
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 805 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 805 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870928, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x1a8e9a0, rem_ep = 0x14898a0, src qpn = 37182, length = 120, local buffer # = 16
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1707a40 on ep 0x1a8e9a0
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
running message thread[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0x1d09460 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
transitioning QP 0x1d0b6e0 to RTS[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],1] for tag 2
transitioning QP 0x1d0bd20 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
successfully set RTS[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
transitioning QP 0x1d0c360 to RTS[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]

[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
successfully set RTS[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],1] for tag 2
sending request for endpoint 0x1a8e9a0[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],1] for tag 2
activating timeout for message 0x1d2d4e0[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870933, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x14898a0, rem_ep = 0x1a8e9a0, src qpn = 37962, length = 120, local buffer # = 21
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1d2d4e0 on ep 0x14898a0
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870934, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x14898a0, rem_ep = 0x1a8e9a0, src qpn = 37962, length = 88, local buffer # = 22
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1d2d6e0 on ep 0x14898a0
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
created message 0x1d2d4e0 with type 100[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2

[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1400 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
activating timeout for message 0x1d2d6e0[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
created message 0x1d2d6e0 with type 101[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1400
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1400 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 1400 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870929, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1d2d4e0 from slid 0x0003 qp 0x0000913e
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870930, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1d2d6e0 from slid 0x0003 qp 0x0000913e
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],1] for tag 2
exiting message thread[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1704460 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1706760 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1706de0 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x1707400 to RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x1728660
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x1728660 with type 101
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0x14898a0.
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870935, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913e, src_qp: 0x0000944a, wc_flags: 0x0, slid: 0x0004
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1728660 from slid 0x0004 qp 0x0000944a
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14225] progress: event_users_decrement setting count to -3
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 804 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 804
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 804 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 804 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870931, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944a, src_qp: 0x0000913e, wc_flags: 0x0, slid: 0x0003
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x1a8e9a0, rem_ep = 0x14898a0, src qpn = 37182, length = 88, local buffer # = 19
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1728660 on ep 0x1a8e9a0
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],1] for tag 2
endpoint 0xa667c0 (lid 4, ep index 3)[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
sending request for endpoint 0xa667c0[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 801
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]

[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 801 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 801 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
activating timeout for message 0xcbc3a0[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870932, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xf08900, rem_ep = 0xa667c0, src qpn = 37963, length = 120, local buffer # = 20
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xcbc3a0 on ep 0xf08900
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 801 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],1] for tag 2
created message 0xcbc3a0 with type 100[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],1] for tag 2
[cn2:01046] progress: event_users_increment setting count to 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870928, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xcbc3a0 from slid 0x0007 qp 0x00008192
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
running message thread[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 74 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2

[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
transitioning QP 0x114ff60 to RTS[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
transitioning QP 0x11521e0 to RTS[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2

[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0x1152820 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
transitioning QP 0x1152e60 to RTS[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]

[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
sending request for endpoint 0xf08900[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2

[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
activating timeout for message 0x1173fe0[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
created message 0x1173fe0 with type 100[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 492 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
running message thread[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 340 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9

[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870933, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1173fe0 from slid 0x0004 qp 0x0000944b
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 74
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 74 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 74 bytes from [[15336,0],1] for tag 2
finishing connection for endpoint 0x1a8e9a0.[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
activating timeout for message 0x11741e0[cn2:01045] progress: event_users_decrement setting count to -2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]

[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
exiting message thread[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2

[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 492
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 492 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 492 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 340
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 340 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 609 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870929, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] [cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
created message 0x11741e0 with type 101[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 340 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
received message. type: 100, lcl_ep = 0xa667c0, rem_ep = 0xf08900, src qpn = 33170, length = 120, local buffer # = 17
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1173fe0 on ep 0xa667c0
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]

[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 222 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870934, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x11741e0 from slid 0x0004 qp 0x0000944b
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
exiting message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 609
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 609 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 609 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870930, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xa667c0, rem_ep = 0xf08900, src qpn = 33170, length = 88, local buffer # = 18
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 222
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 222 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 222 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x11741e0 on ep 0xa667c0
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
running message thread[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]

[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],1] for tag 2
transitioning QP 0xcb8ce0 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2

[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
successfully set RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
transitioning QP 0xcbb0a0 to RTS[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
successfully set RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]

[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],1] for tag 2
transitioning QP 0xcbb740 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
successfully set RTS[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 800
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
transitioning QP 0xcbbd60 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2

[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
successfully set RTS[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 800 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 800 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 800 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]

[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870935, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008192, src_qp: 0x0000944b, wc_flags: 0x0, slid: 0x0004
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xf08900, rem_ep = 0xa667c0, src qpn = 37963, length = 88, local buffer # = 23
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xce1aa0 on ep 0xf08900
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],1] for tag 2
activating timeout for message 0xce1aa0[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2

[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
created message 0xce1aa0 with type 101[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
endpoint 0x860060 (lid 7, ep index 5)[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]

[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
sending request for endpoint 0x860060[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
activating timeout for message 0xa63b00[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 341
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 341 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
created message 0xa63b00 with type 100[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2

[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 490 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 341 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 341 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870924, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn3:21420] progress: event_users_increment setting count to 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 490
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 490 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 490 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],2] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xe77d40, rem_ep = 0x860060, src qpn = 33171, length = 120, local buffer # = 12
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xa63b00 on ep 0xe77d40
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870928, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xa63b00 from slid 0x0005 qp 0x00007e47
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
running message thread[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],2] for tag 2
finishing connection for endpoint 0xf08900.[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
running message thread
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21419] progress: event_users_decrement setting count to -2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
exiting message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0x107c200 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
successfully set RTS[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2

[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0x107e480 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
successfully set RTS[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2

[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
transitioning QP 0x107eb20 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]

[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],3] for tag 2
transitioning QP 0x107f140 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],3] for tag 2
sending request for endpoint 0xe77d40[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
activating timeout for message 0x109d860[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],3] for tag 2
created message 0x109d860 with type 100[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 492

[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 492 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 492 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 492 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870929, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] [cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 340 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 340
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 340 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 340 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870925, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x109d860 from slid 0x0007 qp 0x00008193
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
received message. type: 100, lcl_ep = 0x860060, rem_ep = 0xe77d40, src qpn = 32327, length = 120, local buffer # = 17
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x109d860 on ep 0x860060
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
activating timeout for message 0x109da60[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2

[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],3] for tag 2
created message 0x109da60 with type 101[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 492
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 492 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 339 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 492 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 492 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 339
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 339 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870926, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x109da60 from slid 0x0007 qp 0x00008193
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870930, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] [cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 339 bytes from [[15336,0],2] for tag 2
exiting message threadreceived message. type: 101, lcl_ep = 0x860060, rem_ep = 0xe77d40, src qpn = 32327, length = 88, local buffer # = 18
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x109da60 on ep 0x860060
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
running message thread[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0xa605a0 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
successfully set RTS[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0xa62840 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
transitioning QP 0xa62e80 to RTS[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
successfully set RTS[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
transitioning QP 0xa634c0 to RTS[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2

[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
successfully set RTS[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
activating timeout for message 0xa846e0[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],2] for tag 2
created message 0xa846e0 with type 101[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 800
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870931, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00008193, src_qp: 0x00007e47, wc_flags: 0x0, slid: 0x0005
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xa846e0 from slid 0x0005 qp 0x00007e47
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 800 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 800 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 800 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870927, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e47, src_qp: 0x00008193, wc_flags: 0x0, slid: 0x0007
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xe77d40, rem_ep = 0x860060, src qpn = 33171, length = 88, local buffer # = 15
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xa846e0 on ep 0xe77d40
finishing connection for endpoint 0x860060.[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21420] progress: event_users_decrement setting count to -1
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
endpoint 0xdfd720 (lid 5, ep index 7)[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],2] for tag 2

exiting message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]

[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
sending request for endpoint 0xdfd720[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2

[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
activating timeout for message 0xfd9120[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
created message 0xfd9120 with type 100[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2

[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],3] for tag 2
[cn4:17832] progress: event_users_increment setting count to 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 803
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870924, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xfd9120 from slid 0x0008 qp 0x00006b97
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 803 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 803 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870924, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0x11beb60, rem_ep = 0xdfd720, src qpn = 32328, length = 120, local buffer # = 12
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xfd9120 on ep 0x11beb60
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 803 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
running message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]

[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],3] for tag 2
running message thread[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870931, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000944b, src_qp: 0x00008192, wc_flags: 0x0, slid: 0x0007
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xce1aa0 from slid 0x0007 qp 0x00008192
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
finishing connection for endpoint 0xa667c0.[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224


[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn2:01046] progress: event_users_decrement setting count to -1
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
exiting message thread[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP

[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0x13aa700 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
finishing connection for endpoint 0xe77d40.[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn4:17831] progress: event_users_decrement setting count to -2
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],3] for tag 2
successfully set RTS[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
exiting message thread[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],4] for tag 2
transitioning QP 0x13ac980 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]

[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
successfully set RTS[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]

[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],4] for tag 2
transitioning QP 0x13acfc0 to RTS[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]

[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 63
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 63 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 63 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
transitioning QP 0x13ad600 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 63 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
sending request for endpoint 0x11beb60[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],4] for tag 2
activating timeout for message 0x13d5320[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
created message 0x13d5320 with type 100[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 492
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 492 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870925, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x13d5320 from slid 0x0005 qp 0x00007e48
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 70
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 70 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 341 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 492 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 70 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 492 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
activating timeout for message 0x13d5520[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870925, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] [cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 70 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 341
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 341 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 341 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
received message. type: 100, lcl_ep = 0xdfd720, rem_ep = 0x11beb60, src qpn = 27543, length = 120, local buffer # = 13
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x13d5320 on ep 0xdfd720
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
created message 0x13d5520 with type 101[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 492
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 715
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 492 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 492 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 492 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870926, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 715 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 340
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 715 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 340 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 715 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0] message received 340 bytes from [[15336,0],3] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870926, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x13d5520 from slid 0x0005 qp 0x00007e48
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
received message. type: 101, lcl_ep = 0xdfd720, rem_ep = 0x11beb60, src qpn = 27543, length = 88, local buffer # = 14
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x13d5520 on ep 0xdfd720
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
exiting message thread[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 340 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
running message thread[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
transitioning QP 0xfd5a60 to RTS[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
successfully set RTS[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
transitioning QP 0xfd7e20 to RTS[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
successfully set RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],3] for tag 2
transitioning QP 0xfd84c0 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]

[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
transitioning QP 0xfd8ae0 to RTS[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
successfully set RTS[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],3] for tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],3] for tag 2
activating timeout for message 0xff7420[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP

[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
created message 0xff7420 with type 101[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 802
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 802 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 802 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 802 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870927, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b97, src_qp: 0x00007e48, wc_flags: 0x0, slid: 0x0005
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0x11beb60, rem_ep = 0xdfd720, src qpn = 32328, length = 88, local buffer # = 15
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xff7420 on ep 0x11beb60
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870927, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00007e48, src_qp: 0x00006b97, wc_flags: 0x0, slid: 0x0008
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xff7420 from slid 0x0008 qp 0x00006b97
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
finishing connection for endpoint 0xdfd720.[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17832] progress: event_users_decrement setting count to -1
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
exiting message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
running message thread[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9

[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 74 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 74
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 74 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 74 bytes from [[15336,0],4] for tag 2
finishing connection for endpoint 0x11beb60.[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14174] progress: event_users_decrement setting count to -2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
exiting message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:652:udcm_module_start_connect] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
endpoint 0xba2880 (lid 8, ep index 0)[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] [cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 67 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 67
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 67 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 67 bytes from [[15336,0],4] for tag 2
sending request for endpoint 0xba2880[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2

[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
activating timeout for message 0xdcba40[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
created message 0xdcba40 with type 100[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870928, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xec3e80, rem_ep = 0xba2880, src qpn = 27544, length = 120, local buffer # = 16
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xdcba40 on ep 0xec3e80
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 93
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 93 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 93 bytes from [[15336,0],4] for tag 2
[cn5:14175] progress: event_users_increment setting count to 0
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870924, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xdcba40 from slid 0x0003 qp 0x0000913d
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 93 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x10e1c20 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x10e3ea0 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x10e4540 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] transitioning QP 0x10e4b60 to RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] successfully set RTS
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1556:udcm_send_request] sending request for endpoint 0xec3e80
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x1101220
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] created message 0x1101220 with type 100
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] activating timeout for message 0x1101420
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870929, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1101220 from slid 0x0008 qp 0x00006b98
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 802
created message 0x1101420 with type 101
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 802 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 802 bytes from [[15336,0],4] for tag 2
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870925, status: 0, opcode: 0x80, byte_len: 78, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 100, lcl_ep = 0xba2880, rem_ep = 0xec3e80, src qpn = 37181, length = 120, local buffer # = 13
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1101220 on ep 0xba2880
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 802 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870930, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0x1101420 from slid 0x0008 qp 0x00006b98
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
exiting message thread
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 801
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 801 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 801 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870926, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xba2880, rem_ep = 0xec3e80, src qpn = 37181, length = 88, local buffer # = 14
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0x1101420 on ep 0xba2880
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 801 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] [cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
running message thread[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
transitioning QP 0xdc8460 to RTS[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
successfully set RTS[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
transitioning QP 0xdca760 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
successfully set RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
transitioning QP 0xdcad80 to RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
successfully set RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1140:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
transitioning QP 0xdcb420 to RTS[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 177
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 177 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 177 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1163:udcm_rc_qp_to_rts] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 50
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 50 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 50 bytes from [[15336,0],4] for tag 2
successfully set RTS[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 177 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 50 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 184
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 184 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 184 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2180:udcm_set_message_timeout] [cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 184 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 69
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 69 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 69 bytes from [[15336,0],4] for tag 2
activating timeout for message 0xdf3200[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 176
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 176 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 176 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1537:udcm_new_message] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 68
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 69 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 176 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 68 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 68 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 68 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870931, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x0000913d, src_qp: 0x00006b98, wc_flags: 0x0, slid: 0x0008
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1887:udcm_process_messages] received message. type: 101, lcl_ep = 0xec3e80, rem_ep = 0xba2880, src qpn = 27544, length = 88, local buffer # = 19
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1637:udcm_send_ack] sending ack for message 0xdf3200 on ep 0xec3e80
created message 0xdf3200 with type 101[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 182
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 182 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 182 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] [cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],4] for tag 2
finishing connection for endpoint 0xba2880.[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 182 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 714 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 714
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 714 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 714 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1841:udcm_process_messages] WC: wr_id: 0x0000000536870927, status: 0, opcode: 0x80, byte_len: 58, imm_data: 0x00000000, qp_num: 0x00006b98, src_qp: 0x0000913d, wc_flags: 0x0, slid: 0x0003
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1654:udcm_handle_ack] got ack for message 0xdf3200 from slid 0x0003 qp 0x0000913d
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1662:udcm_handle_ack] found matching message
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 181
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 181 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14175] progress: event_users_decrement setting count to -1
[cn1:14209] [[15336,0],0] message received 181 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 52
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 52 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 52 bytes from [[15336,0],4] for tag 2
exiting message thread[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 181 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 52 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2042:udcm_message_callback] running message thread
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:1792:udcm_finish_connection] finishing connection for endpoint 0xec3e80.
[cn1:14224] progress: event_users_decrement setting count to -5
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/connect/btl_openib_connect_udcm.c:2088:udcm_message_callback] exiting message thread
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],4] for tag 2
sending rkey 2013306553, rdma_start.lval 7fe8eec54074, pval 0x7fe8eec54074, ival 4005904500 type 1 and sizeof(rdma_hdr) 16
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],4] for tag 2
prior to NTOH received  rkey 2013306553, rdma_start.lval 7fe8eec54074, pval 0x7fe8eec54074, ival 4005904500[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],4] for tag 2
received  rkey 2013306553, rdma_start.lval 7fe8eec54074, pval 0x7fe8eec54074, ival 4005904500[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
prior to NTOH received  rkey 1208004940, rdma_start.lval 7f36b675b074, pval 0x7f36b675b074, ival 3061166196[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],1] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],1] for tag 2
sending rkey 1208004940, rdma_start.lval 7f36b675b074, pval 0x7f36b675b074, ival 3061166196 type 1 and sizeof(rdma_hdr) 16
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2

[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
received  rkey 1208004940, rdma_start.lval 7f36b675b074, pval 0x7f36b675b074, ival 3061166196
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 152
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 152 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 152 bytes from [[15336,0],2] for tag 2
sending rkey 536917919, rdma_start.lval 7f0da2e46074, pval 0x7f0da2e46074, ival 2732875892 type 1 and sizeof(rdma_hdr) 16
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 152 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 136
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 136 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0] message received 136 bytes from [[15336,0],1] for tag 2
prior to NTOH received  rkey 536917919, rdma_start.lval 7f0da2e46074, pval 0x7f0da2e46074, ival 2732875892[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 136 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166

sending rkey 1610653844, rdma_start.lval 7f1cb8ede074, pval 0x7f1cb8ede074, ival 3102597236 type 1 and sizeof(rdma_hdr) 16
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]

[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 122
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 122 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 122 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 122 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
received  rkey 536917919, rdma_start.lval 7f0da2e46074, pval 0x7f0da2e46074, ival 2732875892[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
prior to NTOH received  rkey 1610653844, rdma_start.lval 7f1cb8ede074, pval 0x7f1cb8ede074, ival 3102597236[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2

[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
sending rkey 2013306809, rdma_start.lval 7fe491aac074, pval 0x7fe491aac074, ival 2443886708 type 1 and sizeof(rdma_hdr) 16
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9

[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]

[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
received  rkey 1610653844, rdma_start.lval 7f1cb8ede074, pval 0x7f1cb8ede074, ival 3102597236[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],3] for tag 2
prior to NTOH received  rkey 2013306809, rdma_start.lval 7fe491aac074, pval 0x7fe491aac074, ival 2443886708[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
received  rkey 2013306809, rdma_start.lval 7fe491aac074, pval 0x7fe491aac074, ival 2443886708[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]

[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] sending rkey 1342487353, rdma_start.lval 7f9536625074, pval 0x7f9536625074, ival 912412788 type 1 and sizeof(rdma_hdr) 16

[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 136
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 136 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 136 bytes from [[15336,0],4] for tag 2
prior to NTOH received  rkey 1342487353, rdma_start.lval 7f9536625074, pval 0x7f9536625074, ival 912412788[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 136 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 122
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 122 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 122 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
received  rkey 1342487353, rdma_start.lval 7f9536625074, pval 0x7f9536625074, ival 912412788[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 122 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],4] for tag 2
sending rkey 2013307065, rdma_start.lval 7fe491a78074, pval 0x7fe491a78074, ival 2443673716 type 1 and sizeof(rdma_hdr) 16
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],4] for tag 2
prior to NTOH received  rkey 2013307065, rdma_start.lval 7fe491a78074, pval 0x7fe491a78074, ival 2443673716[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],4] for tag 2
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
received  rkey 2013307065, rdma_start.lval 7fe491a78074, pval 0x7fe491a78074, ival 2443673716[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224

[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] sending rkey 1342487609, rdma_start.lval 7fd463ea0074, pval 0x7fd463ea0074, ival 1676279924 type 1 and sizeof(rdma_hdr) 16

[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 186
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 186 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
sending rkey 1208005196, rdma_start.lval 7fe891ef5074, pval 0x7fe891ef5074, ival 2448380020 type 1 and sizeof(rdma_hdr) 16
sending rkey 2013307321, rdma_start.lval 7fe8eec20074, pval 0x7fe8eec20074, ival 4005691508 type 1 and sizeof(rdma_hdr) 16
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 186 bytes from [[15336,0],2] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]

[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_endpoint.c:899:mca_btl_openib_endpoint_send_eager_rdma] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 186 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 504
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 504 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 504 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 153
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 153 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 152 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 152
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 152 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 153 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 504 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 153 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 152 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] prior to NTOH received  rkey 1342487609, rdma_start.lval 7fd463ea0074, pval 0x7fd463ea0074, ival 1676279924
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] received  rkey 1342487609, rdma_start.lval 7fd463ea0074, pval 0x7fd463ea0074, ival 1676279924
sending rkey 1610654100, rdma_start.lval 7f5891d54074, pval 0x7f5891d54074, ival 2446671988 type 1 and sizeof(rdma_hdr) 16
sending rkey 536918175, rdma_start.lval 7f8d8597e074, pval 0x7f8d8597e074, ival 2241323124 type 1 and sizeof(rdma_hdr) 16
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2

[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 272 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2

[cn4:17828] [[15336,0],3] routing direct to the HNP

[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 260 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 272
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 272 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 272 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] prior to NTOH received  rkey 536918175, rdma_start.lval 7f8d8597e074, pval 0x7f8d8597e074, ival 2241323124[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] [cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 137
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 137 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 260
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 260 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 137 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 137 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 260 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
prior to NTOH received  rkey 1208005196, rdma_start.lval 7fe891ef5074, pval 0x7fe891ef5074, ival 2448380020prior to NTOH received  rkey 1610654100, rdma_start.lval 7f5891d54074, pval 0x7f5891d54074, ival 2446671988
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] received  rkey 536918175, rdma_start.lval 7f8d8597e074, pval 0x7f8d8597e074, ival 2241323124
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 166
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 166 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]


[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 166 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 166 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 123 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] [cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
received  rkey 1610654100, rdma_start.lval 7f5891d54074, pval 0x7f5891d54074, ival 2446671988[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2

[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:487:btl_openib_control] prior to NTOH received  rkey 2013307321, rdma_start.lval 7fe8eec20074, pval 0x7fe8eec20074, ival 4005691508
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib_component.c:497:btl_openib_control] received  rkey 2013307321, rdma_start.lval 7fe8eec20074, pval 0x7fe8eec20074, ival 4005691508
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 123
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 123 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 123 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
received  rkey 1208005196, rdma_start.lval 7fe891ef5074, pval 0x7fe891ef5074, ival 2448380020[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2

[cn1:14225] progress: set_event_flag setting to 3
[cn1:14224] progress: set_event_flag setting to 3
[cn1:14224] progress: event_users_increment setting count to -4
[cn1:14224] [[15336,1],0] grpcomm:bad entering barrier
[cn1:14224] [[15336,1],0] ACTIVATING GRCPCOMM OP 2 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn1:14224] [[15336,1],0] rml_send_buffer to peer [[15336,0],0] at tag 30
[cn1:14224] [[15336,1],0] grpcomm:bad barrier underway
[cn1:14224] [[15336,1],0] rml_send_msg to peer [[15336,0],0] at tag 30
[cn1:14224] [[15336,1],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14225] progress: event_users_increment setting count to -2
[cn1:14224] [[15336,1],0] oob:base:send to target [[15336,0],0]
[cn1:14224] [[15336,1],0] oob:base:send known transport for peer [[15336,0],0]
[cn1:14224] [[15336,1],0] oob:tcp:send_nb to peer [[15336,0],0]:30
[cn1:14224] [[15336,1],0] tcp:send_nb to peer [[15336,0],0]
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:30
[cn1:14224] [[15336,1],0] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14224] [[15336,1],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],0]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 10
[cn1:14224] [[15336,1],0] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14224] [[15336,1],0] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14224] [[15336,1],0] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 10 BYTES ON SOCKET 11
[cn1:14224] [[15336,1],0]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14225] [[15336,1],1] grpcomm:bad entering barrier
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14225] [[15336,1],1] ACTIVATING GRCPCOMM OP 2 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 143
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 348
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 348 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] progress: set_event_flag setting to 3
[cn3:21419] progress: set_event_flag setting to 3
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 348
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 348 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 143 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 348 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 348 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 143 bytes from [[15336,0],3] for tag 2
[cn3:21420] progress: set_event_flag setting to 3
[cn3:21420] progress: event_users_increment setting count to 0
[cn3:21420] [[15336,1],5] grpcomm:bad entering barrier
[cn3:21420] [[15336,1],5] ACTIVATING GRCPCOMM OP 2 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01046] progress: set_event_flag setting to 3
[cn2:01046] progress: event_users_increment setting count to 0
[cn2:01046] [[15336,1],3] grpcomm:bad entering barrier
[cn2:01046] [[15336,1],3] ACTIVATING GRCPCOMM OP 2 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14174] progress: set_event_flag setting to 3
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17832] progress: set_event_flag setting to 3
[cn4:17832] progress: event_users_increment setting count to 0
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1366
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1366 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 348
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 348 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 143 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 80
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 80 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],5]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 348 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1366
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],9]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1366 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 1366 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] message received 348 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 10
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,1],5] (ORIGIN [[15336,1],5]) OF 10 BYTES FOR DEST [[15336,0],2] TAG 30
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn1:14209] [[15336,0],0] message received 80 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0] message received 1366 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 10
[cn2:01046] [[15336,1],3] rml_send_buffer to peer [[15336,0],1] at tag 30
[cn2:01046] [[15336,1],3] grpcomm:bad barrier underway
[cn2:01046] [[15336,1],3] rml_send_msg to peer [[15336,0],1] at tag 30
[cn2:01046] [[15336,1],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01046] [[15336,1],3] oob:base:send to target [[15336,0],1]
[cn2:01046] [[15336,1],3] oob:base:send known transport for peer [[15336,0],1]
[cn2:01046] [[15336,1],3] oob:tcp:send_nb to peer [[15336,0],1]:30
[cn2:01046] [[15336,1],3] tcp:send_nb to peer [[15336,0],1]
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:30
[cn2:01046] [[15336,1],3] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn2:01046] [[15336,1],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_handler called to send to peer [[15336,0],1]
[cn2:01046] [[15336,1],3] tcp:send_handler SENDING TO [[15336,0],1]
[cn5:14175] progress: set_event_flag setting to 3
[cn5:14175] progress: event_users_increment setting count to 0
[cn5:14175] [[15336,1],9] grpcomm:bad entering barrier
[cn5:14175] [[15336,1],9] ACTIVATING GRCPCOMM OP 2 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn4:17831] progress: set_event_flag setting to 3
[cn3:21420] [[15336,1],5] rml_send_buffer to peer [[15336,0],2] at tag 30
[cn3:21420] [[15336,1],5] grpcomm:bad barrier underway
[cn3:21420] [[15336,1],5] rml_send_msg to peer [[15336,0],2] at tag 30
[cn3:21420] [[15336,1],5] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21420] [[15336,1],5] oob:base:send to target [[15336,0],2]
[cn3:21420] [[15336,1],5] oob:base:send known transport for peer [[15336,0],2]
[cn3:21420] [[15336,1],5] oob:tcp:send_nb to peer [[15336,0],2]:30
[cn3:21420] [[15336,1],5] tcp:send_nb to peer [[15336,0],2]
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:30
[cn3:21420] [[15336,1],5] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_nb: already connected to [[15336,0],2] - queueing for send
[cn3:21420] [[15336,1],5]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_handler called to send to peer [[15336,0],2]
[cn3:21420] [[15336,1],5] tcp:send_handler SENDING TO [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 275
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 275 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 10
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,1],3] (ORIGIN [[15336,1],3]) OF 10 BYTES FOR DEST [[15336,0],1] TAG 30
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1366
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,1],9] (ORIGIN [[15336,1],9]) OF 10 BYTES FOR DEST [[15336,0],4] TAG 30
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1366 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1571
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 348 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] message received 10 bytes from [[15336,1],5] for tag 30
[cn3:21416] [[15336,0],2] COLLECTIVE RECVD FROM [[15336,1],5]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1571 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 275
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 275 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] message received 10 bytes from [[15336,1],3] for tag 30
[cn1:14209] [[15336,0],0] message received 275 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 1366 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],7]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] message received 1571 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] COLLECTIVE RECVD FROM [[15336,1],3]
[cn1:14209] [[15336,0],0] message received 275 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 348 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01046] [[15336,1],3] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 10 BYTES ON SOCKET 11
[cn2:01046] [[15336,1],3]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14175] [[15336,1],9] rml_send_buffer to peer [[15336,0],4] at tag 30
[cn5:14175] [[15336,1],9] grpcomm:bad barrier underway
[cn5:14175] [[15336,1],9] rml_send_msg to peer [[15336,0],4] at tag 30
[cn5:14175] [[15336,1],9] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14175] [[15336,1],9] oob:base:send to target [[15336,0],4]
[cn5:14175] [[15336,1],9] oob:base:send known transport for peer [[15336,0],4]
[cn5:14175] [[15336,1],9] oob:tcp:send_nb to peer [[15336,0],4]:30
[cn5:14175] [[15336,1],9] tcp:send_nb to peer [[15336,0],4]
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:30
[cn5:14175] [[15336,1],9] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_nb: already connected to [[15336,0],4] - queueing for send
[cn5:14175] [[15336,1],9]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_handler called to send to peer [[15336,0],4]
[cn5:14175] [[15336,1],9] tcp:send_handler SENDING TO [[15336,0],4]
[cn4:17832] [[15336,1],7] grpcomm:bad entering barrier
[cn4:17832] [[15336,1],7] ACTIVATING GRCPCOMM OP 2 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn4:17832] [[15336,1],7] rml_send_buffer to peer [[15336,0],3] at tag 30
[cn4:17832] [[15336,1],7] grpcomm:bad barrier underway
[cn4:17832] [[15336,1],7] rml_send_msg to peer [[15336,0],3] at tag 30
[cn4:17832] [[15336,1],7] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17832] [[15336,1],7] oob:base:send to target [[15336,0],3]
[cn4:17832] [[15336,1],7] oob:base:send known transport for peer [[15336,0],3]
[cn4:17832] [[15336,1],7] oob:tcp:send_nb to peer [[15336,0],3]:30
[cn4:17832] [[15336,1],7] tcp:send_nb to peer [[15336,0],3]
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:30
[cn4:17832] [[15336,1],7] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_nb: already connected to [[15336,0],3] - queueing for send
[cn4:17832] [[15336,1],7]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_handler called to send to peer [[15336,0],3]
[cn4:17832] [[15336,1],7] tcp:send_handler SENDING TO [[15336,0],3]
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn3:21420] [[15336,1],5] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 10 BYTES ON SOCKET 11
[cn3:21420] [[15336,1],5]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] WORKING COLLECTIVE 2
[cn2:01042] [[15336,0],1] PROGRESSING COLLECTIVE 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] message received 10 bytes from [[15336,1],9] for tag 30
[cn5:14171] [[15336,0],4] COLLECTIVE RECVD FROM [[15336,1],9]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 10
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,1],7] (ORIGIN [[15336,1],7]) OF 10 BYTES FOR DEST [[15336,0],3] TAG 30
[cn2:01042] [[15336,0],1] COLL 0 IS LOCALLY COMPLETE
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 1
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] COLL 1 IS LOCALLY COMPLETE
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] WORKING COLLECTIVE 2
[cn5:14171] [[15336,0],4] PROGRESSING COLLECTIVE 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 0
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] WORKING COLLECTIVE 2
[cn3:21416] [[15336,0],2] PROGRESSING COLLECTIVE 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 0
[cn3:21416] [[15336,0],2] COLL 0 IS LOCALLY COMPLETE
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] COLL 1 IS LOCALLY COMPLETE
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 275
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 275 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] COLL 0 IS LOCALLY COMPLETE
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 1
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] COLL 1 IS LOCALLY COMPLETE
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 2
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 275
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 275 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 80 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 275 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 275 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] message received 10 bytes from [[15336,1],7] for tag 30
[cn4:17828] [[15336,0],3] COLLECTIVE RECVD FROM [[15336,1],7]
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] WORKING COLLECTIVE 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 85
[cn4:17828] [[15336,0],3] PROGRESSING COLLECTIVE 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 85 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1366 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] progress: event_users_increment setting count to -1
[cn5:14175] [[15336,1],9] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 10 BYTES ON SOCKET 11
[cn5:14175] [[15336,1],9]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17832] [[15336,1],7] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 10 BYTES ON SOCKET 11
[cn4:17832] [[15336,1],7]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21419] progress: event_users_increment setting count to -1
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 0
[cn4:17828] [[15336,0],3] COLL 0 IS LOCALLY COMPLETE
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 85
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 1
[cn4:17828] [[15336,0],3] COLL 1 IS LOCALLY COMPLETE
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 85 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 2
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 275 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1366 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1366 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 94
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 94 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 85 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 85 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 275 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 94 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 275 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 180
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 180 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21419] [[15336,1],4] grpcomm:bad entering barrier
[cn2:01045] [[15336,1],2] grpcomm:bad entering barrier
[cn5:14174] progress: event_users_increment setting count to -1
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17831] progress: event_users_increment setting count to -1
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 180
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 180 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1571 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 85
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 85 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 85
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 275 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 85 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 180 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 180 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 85 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 85 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 85 bytes from [[15336,0],3] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21419] [[15336,1],4] ACTIVATING GRCPCOMM OP 2 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01045] [[15336,1],2] ACTIVATING GRCPCOMM OP 2 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn5:14174] [[15336,1],8] grpcomm:bad entering barrier
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17831] [[15336,1],6] grpcomm:bad entering barrier
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1611
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 1611 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 180
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 180 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 94 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 180
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 180 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 85 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1611
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 85 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 1611 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 1611 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 180 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 180 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 1611 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 85 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 32
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 180 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 32 BYTES FOR DEST [[15336,0],0] TAG 32
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 180 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 180 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1611
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 180 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],6]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 1611 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 10
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,1],6] (ORIGIN [[15336,1],6]) OF 10 BYTES FOR DEST [[15336,0],3] TAG 30
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] message received 10 bytes from [[15336,1],6] for tag 30
[cn4:17828] [[15336,0],3] COLLECTIVE RECVD FROM [[15336,1],6]
[cn4:17828] [[15336,0],3] WORKING COLLECTIVE 2
[cn4:17828] [[15336,0],3] PROGRESSING COLLECTIVE 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 0
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],2]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 10
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,1],2] (ORIGIN [[15336,1],2]) OF 10 BYTES FOR DEST [[15336,0],1] TAG 30
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] message received 10 bytes from [[15336,1],2] for tag 30
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 1611
[cn4:17828] [[15336,0],3] COLL 0 IS LOCALLY COMPLETE
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 1
[cn4:17828] [[15336,0],3] COLL 1 IS LOCALLY COMPLETE
[cn4:17828] [[15336,0],3] PROGRESSING COLL id 2
[cn4:17828] [[15336,0],3] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn4:17828] [[15336,0],3] COLLECTIVE 2 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],3] at tag 32
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 1611 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] COLLECTIVE RECVD FROM [[15336,1],2]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] WORKING COLLECTIVE 2
[cn2:01042] [[15336,0],1] PROGRESSING COLLECTIVE 2
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 0
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] COLL 0 IS LOCALLY COMPLETE
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 1
[cn2:01042] [[15336,0],1] COLL 1 IS LOCALLY COMPLETE
[cn2:01042] [[15336,0],1] PROGRESSING COLL id 2
[cn2:01042] [[15336,0],1] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],3] at tag 32
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_iovec_to_self at tag 32
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_to_self callback executing for tag 32
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 32
[cn4:17828] [[15336,0],3] message received 32 bytes from [[15336,0],3] for tag 32
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],3]
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: WORKING COLLECTIVE 2
[cn2:01042] [[15336,0],1] COLLECTIVE 2 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 32 BYTES FOR DEST [[15336,0],0] TAG 32
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],1] at tag 32
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],1] at tag 32
[cn2:01042] [[15336,0],1] rml_send_iovec_to_self at tag 32
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn2:01045] [[15336,1],2] rml_send_buffer to peer [[15336,0],1] at tag 30
[cn2:01045] [[15336,1],2] grpcomm:bad barrier underway
[cn2:01045] [[15336,1],2] rml_send_msg to peer [[15336,0],1] at tag 30
[cn2:01045] [[15336,1],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01045] [[15336,1],2] oob:base:send to target [[15336,0],1]
[cn2:01045] [[15336,1],2] oob:base:send known transport for peer [[15336,0],1]
[cn2:01045] [[15336,1],2] oob:tcp:send_nb to peer [[15336,0],1]:30
[cn2:01045] [[15336,1],2] tcp:send_nb to peer [[15336,0],1]
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:30
[cn2:01045] [[15336,1],2] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn2:01045] [[15336,1],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_handler called to send to peer [[15336,0],1]
[cn2:01045] [[15336,1],2] tcp:send_handler SENDING TO [[15336,0],1]
[cn2:01045] [[15336,1],2] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 10 BYTES ON SOCKET 11
[cn2:01045] [[15336,1],2]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14174] [[15336,1],8] ACTIVATING GRCPCOMM OP 2 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],8]
[cn4:17831] [[15336,1],6] ACTIVATING GRCPCOMM OP 2 at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/grpcomm/bad/grpcomm_bad_module.c:208
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn3:21419] [[15336,1],4] rml_send_buffer to peer [[15336,0],2] at tag 30
[cn3:21419] [[15336,1],4] grpcomm:bad barrier underway
[cn3:21419] [[15336,1],4] rml_send_msg to peer [[15336,0],2] at tag 30
[cn3:21419] [[15336,1],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21419] [[15336,1],4] oob:base:send to target [[15336,0],2]
[cn3:21419] [[15336,1],4] oob:base:send known transport for peer [[15336,0],2]
[cn3:21419] [[15336,1],4] oob:tcp:send_nb to peer [[15336,0],2]:30
[cn3:21419] [[15336,1],4] tcp:send_nb to peer [[15336,0],2]
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:30
[cn3:21419] [[15336,1],4] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_nb: already connected to [[15336,0],2] - queueing for send
[cn3:21419] [[15336,1],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_handler called to send to peer [[15336,0],2]
[cn3:21419] [[15336,1],4] tcp:send_handler SENDING TO [[15336,0],2]
[cn3:21419] [[15336,1],4] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 10 BYTES ON SOCKET 11
[cn3:21419] [[15336,1],4]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 10
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,1],8] (ORIGIN [[15336,1],8]) OF 10 BYTES FOR DEST [[15336,0],4] TAG 30
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn1:14209] [[15336,0],0] message received 32 bytes from [[15336,0],1] for tag 32
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] message received 10 bytes from [[15336,1],8] for tag 30
[cn5:14171] [[15336,0],4] COLLECTIVE RECVD FROM [[15336,1],8]
[cn5:14171] [[15336,0],4] WORKING COLLECTIVE 2
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],1]
[cn5:14171] [[15336,0],4] PROGRESSING COLLECTIVE 2
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 0
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 2
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],4]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 10
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,1],4] (ORIGIN [[15336,1],4]) OF 10 BYTES FOR DEST [[15336,0],2] TAG 30
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 4
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: CANNOT PROGRESS
[cn3:21416] [[15336,0],2] message received 10 bytes from [[15336,1],4] for tag 30
[cn3:21416] [[15336,0],2] COLLECTIVE RECVD FROM [[15336,1],4]
[cn3:21416] [[15336,0],2] WORKING COLLECTIVE 2
[cn3:21416] [[15336,0],2] PROGRESSING COLLECTIVE 2
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 0
[cn1:14209] [[15336,0],0] message received 1611 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] COLL 0 IS LOCALLY COMPLETE
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 1
[cn5:14171] [[15336,0],4] COLL 1 IS LOCALLY COMPLETE
[cn5:14171] [[15336,0],4] PROGRESSING COLL id 2
[cn1:14209] [[15336,0],0] message received 1611 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] COLL 0 IS LOCALLY COMPLETE
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 1
[cn5:14171] [[15336,0],4] COLLECTIVE 2 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],4] at tag 32
[cn1:14209] [[15336,0],0] message received 32 bytes from [[15336,0],2] for tag 32
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],4] at tag 32
[cn5:14171] [[15336,0],4] rml_send_iovec_to_self at tag 32
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],2]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: CANNOT PROGRESS
[cn3:21416] [[15336,0],2] COLL 1 IS LOCALLY COMPLETE
[cn3:21416] [[15336,0],2] PROGRESSING COLL id 2
[cn3:21416] [[15336,0],2] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn3:21416] [[15336,0],2] COLLECTIVE 2 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],2] at tag 32
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],2] at tag 32
[cn3:21416] [[15336,0],2] rml_send_iovec_to_self at tag 32
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_to_self callback executing for tag 32
[cn3:21416] [[15336,0],2] message received 32 bytes from [[15336,0],2] for tag 32
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],2]
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: WORKING COLLECTIVE 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn3:21416] [[15336,0],2] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 32
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 32
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 32 BYTES FOR DEST [[15336,0],0] TAG 32
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],1]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],1] at tag 32
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14174] [[15336,1],8] rml_send_buffer to peer [[15336,0],4] at tag 30
[cn5:14174] [[15336,1],8] grpcomm:bad barrier underway
[cn5:14174] [[15336,1],8] rml_send_msg to peer [[15336,0],4] at tag 30
[cn5:14174] [[15336,1],8] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14174] [[15336,1],8] oob:base:send to target [[15336,0],4]
[cn5:14174] [[15336,1],8] oob:base:send known transport for peer [[15336,0],4]
[cn5:14174] [[15336,1],8] oob:tcp:send_nb to peer [[15336,0],4]:30
[cn5:14174] [[15336,1],8] tcp:send_nb to peer [[15336,0],4]
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:30
[cn5:14174] [[15336,1],8] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_nb: already connected to [[15336,0],4] - queueing for send
[cn5:14174] [[15336,1],8]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_handler called to send to peer [[15336,0],4]
[cn5:14174] [[15336,1],8] tcp:send_handler SENDING TO [[15336,0],4]
[cn5:14174] [[15336,1],8] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 10 BYTES ON SOCKET 11
[cn5:14174] [[15336,1],8]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17831] [[15336,1],6] rml_send_buffer to peer [[15336,0],3] at tag 30
[cn4:17831] [[15336,1],6] grpcomm:bad barrier underway
[cn4:17831] [[15336,1],6] rml_send_msg to peer [[15336,0],3] at tag 30
[cn4:17831] [[15336,1],6] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17831] [[15336,1],6] oob:base:send to target [[15336,0],3]
[cn4:17831] [[15336,1],6] oob:base:send known transport for peer [[15336,0],3]
[cn4:17831] [[15336,1],6] oob:tcp:send_nb to peer [[15336,0],3]:30
[cn4:17831] [[15336,1],6] tcp:send_nb to peer [[15336,0],3]
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:30
[cn4:17831] [[15336,1],6] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn4:17831] [[15336,1],6] tcp:send_nb: already connected to [[15336,0],3] - queueing for send
[cn4:17831] [[15336,1],6]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],3]
[cn4:17831] [[15336,1],6] tcp:send_handler called to send to peer [[15336,0],3]
[cn4:17831] [[15336,1],6] tcp:send_handler SENDING TO [[15336,0],3]
[cn4:17831] [[15336,1],6] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 10 BYTES ON SOCKET 11
[cn4:17831] [[15336,1],6]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:32
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 32 bytes from [[15336,0],4] for tag 32
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:32
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],4]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],1] at tag 32
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: CANNOT PROGRESS
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],1]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],1]:32
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],1]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1611 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:32
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 32 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1611 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],1]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],1]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 32 BYTES ON SOCKET 10
[cn4:17828] [[15336,0],3]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_to_self callback executing for tag 32
[cn5:14171] [[15336,0],4] message received 32 bytes from [[15336,0],4] for tag 32
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],4]
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: WORKING COLLECTIVE 2
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn5:14171] [[15336,0],4] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 32
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:32
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:32
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1611 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 32 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_to_self callback executing for tag 32
[cn2:01042] [[15336,0],1] message received 32 bytes from [[15336,0],1] for tag 32
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],1]
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: WORKING COLLECTIVE 2
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 1611 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 32
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 32 BYTES FOR DEST [[15336,0],1] TAG 32
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] message received 32 bytes from [[15336,0],3] for tag 32
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],3]
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: WORKING COLLECTIVE 2
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: RELAYING COLLECTIVE TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn2:01042] [[15336,0],1] grpcomm:base:daemon_coll: MISSING CONTRIBUTORS: np 10 ngr 4
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 32
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:32
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:32
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],1]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 32 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 10
[cn1:14225] [[15336,1],1] rml_send_buffer to peer [[15336,0],0] at tag 30
[cn1:14225] [[15336,1],1] grpcomm:bad barrier underway
[cn1:14225] [[15336,1],1] rml_send_msg to peer [[15336,0],0] at tag 30
[cn1:14225] [[15336,1],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14225] [[15336,1],1] oob:base:send to target [[15336,0],0]
[cn1:14225] [[15336,1],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14225] [[15336,1],1] oob:tcp:send_nb to peer [[15336,0],0]:30
[cn1:14225] [[15336,1],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:30
[cn1:14225] [[15336,1],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14225] [[15336,1],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14225] [[15336,1],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14225] [[15336,1],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 10 BYTES ON SOCKET 11
[cn1:14225] [[15336,1],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,1],0] (ORIGIN [[15336,1],0]) OF 10 BYTES FOR DEST [[15336,0],0] TAG 30
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 10 bytes from [[15336,1],0] for tag 30
[cn1:14209] [[15336,0],0] COLLECTIVE RECVD FROM [[15336,1],0]
[cn1:14209] [[15336,0],0] WORKING COLLECTIVE 2
[cn1:14209] [[15336,0],0] PROGRESSING COLLECTIVE 2
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 2
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,1],1] (ORIGIN [[15336,1],1]) OF 10 BYTES FOR DEST [[15336,0],0] TAG 30
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 10 bytes from [[15336,1],1] for tag 30
[cn1:14209] [[15336,0],0] COLLECTIVE RECVD FROM [[15336,1],1]
[cn1:14209] [[15336,0],0] WORKING COLLECTIVE 2
[cn1:14209] [[15336,0],0] PROGRESSING COLLECTIVE 2
[cn1:14209] [[15336,0],0] PROGRESSING COLL id 2
[cn1:14209] [[15336,0],0] ALL LOCAL PROCS FOR JOB [15336,1] CONTRIBUTE 2
[cn1:14209] [[15336,0],0] COLLECTIVE 2 LOCALLY COMPLETE - SENDING TO GLOBAL COLLECTIVE
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 32
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 32
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 32
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 32
[cn1:14209] [[15336,0],0] message received 32 bytes from [[15336,0],0] for tag 32
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: daemon collective recvd from [[15336,0],0]
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: WORKING COLLECTIVE 2
[cn1:14209] [[15336,0],0] grpcomm:base:daemon_coll: NUM CONTRIBS: 2
[cn1:14209] [[15336,0],0] grpcomm:bad:xcast sent to job [15336,1] tag 30
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 15
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 15
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 15
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 15
[cn1:14209] [[15336,0],0] message received 40 bytes from [[15336,0],0] for tag 15
[cn1:14209] [[15336,0],0] grpcomm:xcast:recv: with 40 bytes
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],1]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],1] at tag 15
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],2]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],2] at tag 15
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],4] at tag 15
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 1
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],1] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],2] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],4] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 1
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 1
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],1]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],2]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],4]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],4]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],4]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 1
[cn1:14209] [[15336,0],0] message received 40 bytes from [[15336,0],0] for tag 1
[cn1:14209] [[15336,0],0] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn1:14209] [[15336,0],0] orted_cmd: received message_local_procs
[cn1:14209] [[15336,0],0] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,1],0] at tag 30
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,1],1] at tag 30
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],2] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],2]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],4] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,1],0] at tag 30
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,1],1] at tag 30
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,1],0]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,1],0]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,1],0]:30
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],0]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,1],1]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,1],1]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,1],1]:30
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],0]:30
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,1],0]) --> [[15336,1],0]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,1],0] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],0]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],1]:30
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,1],1]) --> [[15336,1],1]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,1],1] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],1]
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],1]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],1]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 40 BYTES ON SOCKET 14
[cn1:14209] [[15336,0],0]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],4]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 40 BYTES ON SOCKET 15
[cn1:14209] [[15336,0],0]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],2]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],2]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 40 BYTES ON SOCKET 17
[cn1:14209] [[15336,0],0]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,1],0]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,1],0]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,1],0] OF 10 BYTES ON SOCKET 20
[cn1:14209] [[15336,0],0]-[[15336,1],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,1],1]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,1],1]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,1],1] OF 10 BYTES ON SOCKET 0
[cn1:14209] [[15336,0],0]-[[15336,1],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14224] [[15336,1],0]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14224] [[15336,1],0]:tcp:recv:handler CONNECTED
[cn1:14224] [[15336,1],0]:tcp:recv:handler allocate new recv msg
[cn1:14224] [[15336,1],0]:tcp:recv:handler read hdr
[cn1:14224] [[15336,1],0]:tcp:recv:handler allocate data region of size 10
[cn1:14224] [[15336,1],0] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 10 BYTES FOR DEST [[15336,1],0] TAG 30
[cn1:14224] [[15336,1],0] DELIVERING TO RML
[cn1:14224] [[15336,1],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14224] [[15336,1],0] message received 10 bytes from [[15336,0],0] for tag 30
[cn1:14224] [[15336,1],0] grpcomm:base:receive processing collective return for id 2 recvd from [[15336,0],0]
[cn1:14224] [[15336,1],0] CHECKING COLL id 2
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 40
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 40
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 40
[cn1:14225] [[15336,1],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn1:14225] [[15336,1],1]:tcp:recv:handler CONNECTED
[cn1:14225] [[15336,1],1]:tcp:recv:handler allocate new recv msg
[cn1:14225] [[15336,1],1]:tcp:recv:handler read hdr
[cn1:14225] [[15336,1],1]:tcp:recv:handler allocate data region of size 10
[cn1:14225] [[15336,1],1] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 10 BYTES FOR DEST [[15336,1],1] TAG 30
[cn1:14225] [[15336,1],1] DELIVERING TO RML
[cn1:14225] [[15336,1],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14225] [[15336,1],1] message received 10 bytes from [[15336,0],0] for tag 30
[cn1:14225] [[15336,1],1] grpcomm:base:receive processing collective return for id 2 recvd from [[15336,0],0]
[cn1:14225] [[15336,1],1] CHECKING COLL id 2
[cn1][[15336,1],1][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] in del_procs 0, setting another endpoint to null
hellompi: /home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1151: mca_btl_openib_del_procs: Assertion `((opal_object_t*)endpoint)->obj_reference_count == 1' failed.
[cn1][[15336,1],0][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] in del_procs 0, setting another endpoint to null
hellompi: /home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1151: mca_btl_openib_del_procs: Assertion `((opal_object_t*)endpoint)->obj_reference_count == 1' failed.
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]-[[15336,1],1] mca_oob_tcp_msg_recv: peer closed connection
[cn1:14209] [[15336,0],0] tcp_peer_close for [[15336,1],1] sd 0 state CONNECTED
[cn1:14209] [[15336,0],0] tcp:lost connection called for peer [[15336,1],1]
[cn1:14209] [[15336,0],0] route to [[15336,1],1] lost
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,1],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]-[[15336,1],0] mca_oob_tcp_msg_recv: peer closed connection
[cn1:14209] [[15336,0],0] tcp_peer_close for [[15336,1],0] sd 20 state CONNECTED
[cn1:14209] [[15336,0],0] tcp:lost connection called for peer [[15336,1],0]
[cn1:14209] [[15336,0],0] route to [[15336,1],0] lost
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 40 BYTES FOR DEST [[15336,0],1] TAG 15
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] message received 40 bytes from [[15336,0],0] for tag 15
[cn2:01042] [[15336,0],1] grpcomm:xcast:recv: with 40 bytes
[cn2:01042] [[15336,0],1] orte:daemon:send_relay sending relay msg to [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],3] at tag 15
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],1] at tag 1
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],3] at tag 15
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],1] at tag 1
[cn2:01042] [[15336,0],1] rml_send_iovec_to_self at tag 1
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],3]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],3]:15
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_to_self callback executing for tag 1
[cn2:01042] [[15336,0],1] message received 40 bytes from [[15336,0],1] for tag 1
[cn2:01042] [[15336,0],1] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn2:01042] [[15336,0],1] orted_cmd: received message_local_procs
[cn2:01042] [[15336,0],1] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,1],2] at tag 30
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,1],3] at tag 30
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:15
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],3] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,1],2] at tag 30
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,1],3] at tag 30
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,1],2]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,1],2]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,1],2]:30
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,1],2]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],2]
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,1],3]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,1],3]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,1],3]:30
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,1],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],2]:30
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,1],2]) --> [[15336,1],2]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,1],2] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],2]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],3]:30
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,1],3]) --> [[15336,1],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,1],3] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],3]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,1],2]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,1],2]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,1],2] OF 10 BYTES ON SOCKET 13
[cn2:01042] [[15336,0],1]-[[15336,1],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,1],3]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,1],3]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,1],3] OF 10 BYTES ON SOCKET 14
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 525
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 40
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 525 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 525 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,1],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],3]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 40 BYTES ON SOCKET 10
[cn2:01042] [[15336,0],1]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01046] [[15336,1],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01046] [[15336,1],3]:tcp:recv:handler CONNECTED
[cn2:01046] [[15336,1],3]:tcp:recv:handler allocate new recv msg
[cn2:01046] [[15336,1],3]:tcp:recv:handler read hdr
[cn2:01046] [[15336,1],3]:tcp:recv:handler allocate data region of size 10
[cn2:01046] [[15336,1],3] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 10 BYTES FOR DEST [[15336,1],3] TAG 30
[cn2:01046] [[15336,1],3] DELIVERING TO RML
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 401
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 401 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 401 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 525 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 896
[cn2:01046] [[15336,1],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01046] [[15336,1],3] message received 10 bytes from [[15336,0],1] for tag 30
[cn2:01046] [[15336,1],3] grpcomm:base:receive processing collective return for id 2 recvd from [[15336,0],1]
[cn2:01046] [[15336,1],3] CHECKING COLL id 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 401 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 40 BYTES FOR DEST [[15336,0],3] TAG 15
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] message received 40 bytes from [[15336,0],1] for tag 15
[cn4:17828] [[15336,0],3] grpcomm:xcast:recv: with 40 bytes
[cn4:17828] [[15336,0],3] orte:daemon:send_relay - recipient list is empty!
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],3] at tag 1
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],3] at tag 1
[cn4:17828] [[15336,0],3] rml_send_iovec_to_self at tag 1
[cn4:17828] [[15336,0],3] rml_send_to_self callback executing for tag 1
[cn4:17828] [[15336,0],3] message received 40 bytes from [[15336,0],3] for tag 1
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 896 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 896 bytes from [[15336,0],1] for tag 2
[cn2:01045] [[15336,1],2]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01045] [[15336,1],2]:tcp:recv:handler CONNECTED
[cn2:01045] [[15336,1],2]:tcp:recv:handler allocate new recv msg
[cn2:01045] [[15336,1],2]:tcp:recv:handler read hdr
[cn2:01045] [[15336,1],2]:tcp:recv:handler allocate data region of size 10
[cn2:01045] [[15336,1],2] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 10 BYTES FOR DEST [[15336,1],2] TAG 30
[cn2:01045] [[15336,1],2] DELIVERING TO RML
[cn2:01045] [[15336,1],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01045] [[15336,1],2] message received 10 bytes from [[15336,0],1] for tag 30
[cn2:01045] [[15336,1],2] grpcomm:base:receive processing collective return for id 2 recvd from [[15336,0],1]
[cn2:01045] [[15336,1],2] CHECKING COLL id 2
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 40 BYTES FOR DEST [[15336,0],4] TAG 15
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] message received 40 bytes from [[15336,0],0] for tag 15
[cn5:14171] [[15336,0],4] grpcomm:xcast:recv: with 40 bytes
[cn5:14171] [[15336,0],4] orte:daemon:send_relay - recipient list is empty!
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],4] at tag 1
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],4] at tag 1
[cn5:14171] [[15336,0],4] rml_send_iovec_to_self at tag 1
[cn5:14171] [[15336,0],4] rml_send_to_self callback executing for tag 1
[cn5:14171] [[15336,0],4] message received 40 bytes from [[15336,0],4] for tag 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn4:17828] [[15336,0],3] orted_cmd: received message_local_procs
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,1],6] at tag 30
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,1],7] at tag 30
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,1],6] at tag 30
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 348
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,1],7] at tag 30
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,1],6]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,1],6]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,1],6]:30
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,1],6]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],6]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,1],7]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 896 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,1],7]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,1],7]:30
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,1],7]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],7]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],6]:30
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,1],6]) --> [[15336,1],6]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,1],6] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],6]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],7]:30
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,1],7]) --> [[15336,1],7]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,1],7] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],7]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,1],6]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,1],6]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,1],6] OF 10 BYTES ON SOCKET 13
[cn4:17828] [[15336,0],3]-[[15336,1],6] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,1],7]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,1],7]
[cn5:14171] [[15336,0],4] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn5:14171] [[15336,0],4] orted_cmd: received message_local_procs
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,1],7] OF 10 BYTES ON SOCKET 14
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 348 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,1],8] at tag 30
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,1],9] at tag 30
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,1],8] at tag 30
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,1],9] at tag 30
[cn1:14209] [[15336,0],0] message received 348 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,1],8]:30
[cn4:17828] [[15336,0],3]-[[15336,1],7] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 40 BYTES FOR DEST [[15336,0],2] TAG 15
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] message received 40 bytes from [[15336,0],0] for tag 15
[cn3:21416] [[15336,0],2] grpcomm:xcast:recv: with 40 bytes
[cn3:21416] [[15336,0],2] orte:daemon:send_relay - recipient list is empty!
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],2] at tag 1
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],2] at tag 1
[cn3:21416] [[15336,0],2] rml_send_iovec_to_self at tag 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_to_self callback executing for tag 1
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 578
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,1],8]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],8]
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,1],9]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,1],9]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,1],9]:30
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,1],9]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 578 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],9]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],8]:30
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,1],8]) --> [[15336,1],8]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,1],8] - queueing for send
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17832] [[15336,1],7]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17832] [[15336,1],7]:tcp:recv:handler CONNECTED
[cn4:17832] [[15336,1],7]:tcp:recv:handler allocate new recv msg
[cn4:17832] [[15336,1],7]:tcp:recv:handler read hdr
[cn4:17832] [[15336,1],7]:tcp:recv:handler allocate data region of size 10
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] message received 40 bytes from [[15336,0],2] for tag 1
[cn3:21416] [[15336,0],2] orted:comm:process_commands() Processing Command: ORTE_DAEMON_MESSAGE_LOCAL_PROCS
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] orted_cmd: received message_local_procs
[cn3:21416] [[15336,0],2] orted:comm:message_local_procs delivering message to job [15336,1] tag 30
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,1],4] at tag 30
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,1],5] at tag 30
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 273
[cn1:14209] [[15336,0],0] message received 578 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],8]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],9]:30
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,1],9]) --> [[15336,1],9]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,1],9] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],9]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17832] [[15336,1],7] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 10 BYTES FOR DEST [[15336,1],7] TAG 30
[cn4:17832] [[15336,1],7] DELIVERING TO RML
[cn4:17832] [[15336,1],7] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17832] [[15336,1],7] message received 10 bytes from [[15336,0],3] for tag 30
[cn4:17832] [[15336,1],7] grpcomm:base:receive processing collective return for id 2 recvd from [[15336,0],3]
[cn4:17832] [[15336,1],7] CHECKING COLL id 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,1],4] at tag 30
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,1],5] at tag 30
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,1],4]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,1],4]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,1],4]:30
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,1],4]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,1],8]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,1],8]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 896
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,1],8] OF 10 BYTES ON SOCKET 12
[cn5:14171] [[15336,0],4]-[[15336,1],8] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,1],9]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 896 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 896 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 163
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 273 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,1],9]
[cn4:17831] [[15336,1],6]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17831] [[15336,1],6]:tcp:recv:handler CONNECTED
[cn4:17831] [[15336,1],6]:tcp:recv:handler allocate new recv msg
[cn4:17831] [[15336,1],6]:tcp:recv:handler read hdr
[cn4:17831] [[15336,1],6]:tcp:recv:handler allocate data region of size 10
[cn4:17831] [[15336,1],6] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 10 BYTES FOR DEST [[15336,1],6] TAG 30
[cn4:17831] [[15336,1],6] DELIVERING TO RML
[cn4:17831] [[15336,1],6] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17831] [[15336,1],6] message received 10 bytes from [[15336,0],3] for tag 30
[cn4:17831] [[15336,1],6] grpcomm:base:receive processing collective return for id 2 recvd from [[15336,0],3]
[cn4:17831] [[15336,1],6] CHECKING COLL id 2
[cn1:14209] [[15336,0],0] message received 273 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14175] [[15336,1],9]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14175] [[15336,1],9]:tcp:recv:handler CONNECTED
[cn5:14175] [[15336,1],9]:tcp:recv:handler allocate new recv msg
[cn5:14175] [[15336,1],9]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,1],9] OF 10 BYTES ON SOCKET 13
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]-[[15336,1],9] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 653
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 653 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 348 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],4]
[cn1:14209] [[15336,0],0] message received 653 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,1],5]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,1],5]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,1],5]:30
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,1],5]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,1],5]
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14175] [[15336,1],9]:tcp:recv:handler allocate data region of size 10
[cn5:14175] [[15336,1],9] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 10 BYTES FOR DEST [[15336,1],9] TAG 30
[cn5:14175] [[15336,1],9] DELIVERING TO RML
[cn5:14175] [[15336,1],9] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14175] [[15336,1],9] message received 10 bytes from [[15336,0],4] for tag 30
[cn5:14175] [[15336,1],9] grpcomm:base:receive processing collective return for id 2 recvd from [[15336,0],4]
[cn5:14175] [[15336,1],9] CHECKING COLL id 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],4]:30
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,1],4]) --> [[15336,1],4]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,1],4] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 163 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],4]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,1],5]:30
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,1],5]) --> [[15336,1],5]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 896
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 896 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,1],5] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,1],5]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 163 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 896 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,1],5]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,1],5]
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] [cn5:14174] [[15336,1],8]:tcp:recv:handler called for peer [[15336,0],4]
[cn5:14174] [[15336,1],8]:tcp:recv:handler CONNECTED
[cn5:14174] [[15336,1],8]:tcp:recv:handler allocate new recv msg
[cn5:14174] [[15336,1],8]:tcp:recv:handler read hdr
[cn5:14174] [[15336,1],8]:tcp:recv:handler allocate data region of size 10
[cn5:14174] [[15336,1],8] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 10 BYTES FOR DEST [[15336,1],8] TAG 30
[cn5:14174] [[15336,1],8] DELIVERING TO RML
[cn5:14174] [[15336,1],8] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14174] [[15336,1],8] message received 10 bytes from [[15336,0],4] for tag 30
[cn5:14174] [[15336,1],8] grpcomm:base:receive processing collective return for id 2 recvd from [[15336,0],4]
[cn5:14174] [[15336,1],8] CHECKING COLL id 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 78
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 78 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 273 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 896
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 896 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 78 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 578 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 163 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 896 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 896
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
in del_procs 0, setting another endpoint to null[cn3:21420] [[15336,1],5]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21420] [[15336,1],5]:tcp:recv:handler CONNECTED
[cn3:21420] [[15336,1],5]:tcp:recv:handler allocate new recv msg
[cn3:21420] [[15336,1],5]:tcp:recv:handler read hdr
[cn3:21420] [[15336,1],5]:tcp:recv:handler allocate data region of size 10
[cn3:21420] [[15336,1],5] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 10 BYTES FOR DEST [[15336,1],5] TAG 30
[cn3:21420] [[15336,1],5] DELIVERING TO RML
[cn3:21420] [[15336,1],5] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21420] [[15336,1],5] message received 10 bytes from [[15336,0],2] for tag 30
[cn3:21420] [[15336,1],5] grpcomm:base:receive processing collective return for id 2 recvd from [[15336,0],2]
[cn3:21420] [[15336,1],5] CHECKING COLL id 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 896 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,1],5] OF 10 BYTES ON SOCKET 13
[cn3:21416] [[15336,0],2]-[[15336,1],5] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,1],4]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,1],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,1],4] OF 10 BYTES ON SOCKET 12
[cn3:21416] [[15336,0],2]-[[15336,1],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 163
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 653 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 163 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]

[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 163 bytes from [[15336,0],1] for tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 896 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2][[15336,1],3][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] [cn1:14209] [[15336,0],0] message received 896 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 896 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 163
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21419] [[15336,1],4]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21419] [[15336,1],4]:tcp:recv:handler CONNECTED
[cn3:21419] [[15336,1],4]:tcp:recv:handler allocate new recv msg
[cn3:21419] [[15336,1],4]:tcp:recv:handler read hdr
[cn3:21419] [[15336,1],4]:tcp:recv:handler allocate data region of size 10
[cn3:21419] [[15336,1],4] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 10 BYTES FOR DEST [[15336,1],4] TAG 30
[cn3:21419] [[15336,1],4] DELIVERING TO RML
[cn3:21419] [[15336,1],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21419] [[15336,1],4] message received 10 bytes from [[15336,0],2] for tag 30
[cn3:21419] [[15336,1],4] grpcomm:base:receive processing collective return for id 2 recvd from [[15336,0],2]
[cn3:21419] [[15336,1],4] CHECKING COLL id 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 896 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 78
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 78 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] message received 78 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 78 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 410
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 410 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
in del_procs 1, setting another endpoint to null[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 410 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2][[15336,1],2][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] in del_procs 0, setting another endpoint to null
hellompi: /home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1151: mca_btl_openib_del_procs: Assertion `((opal_object_t*)endpoint)->obj_reference_count == 1' failed.
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 896 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 163 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 78 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],1] for tag 2
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 228
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 163 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 228 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]

[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 228 bytes from [[15336,0],1] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
hellompi: /home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1151: mca_btl_openib_del_procs: Assertion `((opal_object_t*)endpoint)->obj_reference_count == 1' failed.
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 163
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 163 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 163 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 163 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 410 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 78
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 78 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5][[15336,1],9][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] [cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 78 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 163 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 78 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 163
in del_procs 0, setting another endpoint to null[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 163 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 2
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 78
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 78 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] [cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 78 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 78 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199

[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
in del_procs 0, setting another endpoint to null[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 228
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 228 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 163 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 163 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 228 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 163 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 228 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 163
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 78 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 163 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 78
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 78 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
hellompi: /home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1151: mca_btl_openib_del_procs: Assertion `((opal_object_t*)endpoint)->obj_reference_count == 1' failed.

[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] [cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 163 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 78 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] in del_procs 0, setting another endpoint to null[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 78
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 78 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 228 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 78 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 163 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 163
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 163 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 163 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 212
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 212 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 163
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 163 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
in del_procs 1, setting another endpoint to null[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2

[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 163 bytes from [[15336,0],4] for tag 2
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 212 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 78 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 163 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 78
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 78 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 163 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 592
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 78 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 592 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5][[15336,1],8][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] [cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] in del_procs 0, setting another endpoint to null
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] [cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0] message received 78 bytes from [[15336,0],4] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 592 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 410
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 212 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 410 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
in del_procs 0, setting another endpoint to null[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] in del_procs 1, setting another endpoint to null
[cn4][[15336,1],6][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] in del_procs 2, setting another endpoint to null
hellompi: /home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1151: mca_btl_openib_del_procs: Assertion `((opal_object_t*)endpoint)->obj_reference_count == 1' failed.
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 410 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 2
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 410 bytes from [[15336,0],2] for tag 2
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 163
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 163 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 592 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 78
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 78 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP

[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3][[15336,1],4][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] in del_procs 0, setting another endpoint to null
hellompi: /home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1151: mca_btl_openib_del_procs: Assertion `((opal_object_t*)endpoint)->obj_reference_count == 1' failed.
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 163 bytes from [[15336,0],3] for tag 2
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 78 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 78 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 228 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 78
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 78 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] in del_procs 1, setting another endpoint to null[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]

[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 228
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 163 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 228 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 78 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 78 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] message received 228 bytes from [[15336,0],4] for tag 2
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
in del_procs 2, setting another endpoint to null
hellompi: /home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1151: mca_btl_openib_del_procs: Assertion `((opal_object_t*)endpoint)->obj_reference_count == 1' failed.
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]

[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 163
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 163 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 163 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 163
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 78 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 163 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 163 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 163 bytes from [[15336,0],2] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 2
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 78
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 78 BYTES FOR DEST [[15336,0],0] TAG 2
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4][[15336,1],7][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] [cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] [cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn1:14209] [[15336,0],0] message received 78 bytes from [[15336,0],3] for tag 2
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 228 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],2]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,1],2] mca_oob_tcp_msg_recv: peer closed connection
[cn2:01042] [[15336,0],1] tcp_peer_close for [[15336,1],2] sd 13 state CONNECTED
[cn2:01042] [[15336,0],1] tcp:lost connection called for peer [[15336,1],2]
[cn2:01042] [[15336,0],1] route to [[15336,1],2] lost
in del_procs 3, setting another endpoint to null[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 163 BYTES ON SOCKET 9
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 78
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 78 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 78 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],3] for tag 2
[cn1:14209] [[15336,0],0] message received 78 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 228
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 228 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 31
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 31 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536

in del_procs 2, setting another endpoint to null[cn1:14209] [[15336,0],0] message received 228 bytes from [[15336,0],3] for tag 2
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],9]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,1],9] mca_oob_tcp_msg_recv: peer closed connection
[cn5:14171] [[15336,0],4] tcp_peer_close for [[15336,1],9] sd 13 state CONNECTED
[cn5:14171] [[15336,0],4] tcp:lost connection called for peer [[15336,1],9]
[cn1:14209] [[15336,0],0] message received 31 bytes from [[15336,0],2] for tag 2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
hellompi: /home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1151: mca_btl_openib_del_procs: Assertion `((opal_object_t*)endpoint)->obj_reference_count == 1' failed.
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]

[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 31 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,1],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,1],3] mca_oob_tcp_msg_recv: peer closed connection
[cn2:01042] [[15336,0],1] tcp_peer_close for [[15336,1],3] sd 14 state CONNECTED
[cn2:01042] [[15336,0],1] tcp:lost connection called for peer [[15336,1],3]
[cn2:01042] [[15336,0],1] route to [[15336,1],3] lost
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],6]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4] route to [[15336,1],9] lost
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],4]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,1],4] mca_oob_tcp_msg_recv: peer closed connection
[cn3:21416] [[15336,0],2] tcp_peer_close for [[15336,1],4] sd 12 state CONNECTED
[cn4:17828] [[15336,0],3]-[[15336,1],6] mca_oob_tcp_msg_recv: peer closed connection
[cn4:17828] [[15336,0],3] tcp_peer_close for [[15336,1],6] sd 13 state CONNECTED
[cn4:17828] [[15336,0],3] tcp:lost connection called for peer [[15336,1],6]
[cn4:17828] [[15336,0],3] route to [[15336,1],6] lost
[cn3:21416] [[15336,0],2] tcp:lost connection called for peer [[15336,1],4]
[cn3:21416] [[15336,0],2] route to [[15336,1],4] lost
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,1],8]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]-[[15336,1],8] mca_oob_tcp_msg_recv: peer closed connection
[cn5:14171] [[15336,0],4] tcp_peer_close for [[15336,1],8] sd 12 state CONNECTED
[cn5:14171] [[15336,0],4] tcp:lost connection called for peer [[15336,1],8]
[cn5:14171] [[15336,0],4] route to [[15336,1],8] lost
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 212
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 212 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 2
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:2
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 410 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,1],7]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]-[[15336,1],7] mca_oob_tcp_msg_recv: peer closed connection
[cn4:17828] [[15336,0],3] tcp_peer_close for [[15336,1],7] sd 14 state CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 212 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 212 bytes from [[15336,0],2] for tag 2
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn4:17828] [[15336,0],3] tcp:lost connection called for peer [[15336,1],7]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] route to [[15336,1],7] lost
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 410
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 410 BYTES FOR DEST [[15336,0],0] TAG 2
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] in del_procs 3, setting another endpoint to null
[cn1:14209] [[15336,0],0] message received 410 bytes from [[15336,0],2] for tag 2
[cn3][[15336,1],5][/home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1148:mca_btl_openib_del_procs] in del_procs 4, setting another endpoint to null
hellompi: /home/research/artpol/ompi_dev//ompi-trunk_r31907/ompi/mca/btl/openib/btl_openib.c:1151: mca_btl_openib_del_procs: Assertion `((opal_object_t*)endpoint)->obj_reference_count == 1' failed.
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,1],5]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]-[[15336,1],5] mca_oob_tcp_msg_recv: peer closed connection
[cn3:21416] [[15336,0],2] tcp_peer_close for [[15336,1],5] sd 13 state CONNECTED
[cn3:21416] [[15336,0],2] tcp:lost connection called for peer [[15336,1],5]
[cn3:21416] [[15336,0],2] route to [[15336,1],5] lost
[cn1:14209] [[15336,0],0]:/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/errmgr/default_hnp/errmgr_default_hnp.c(421) updating exit status to 139
[cn1:14209] [[15336,0],0] plm:base:orted_cmd sending orted_exit commands
[cn1:14209] [[15336,0],0] grpcomm:bad:xcast sent to job [15336,0] tag 1
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 15
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 15
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 15
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 15
[cn1:14209] [[15336,0],0] message received 8 bytes from [[15336,0],0] for tag 15
[cn1:14209] [[15336,0],0] grpcomm:xcast:recv: with 8 bytes
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],1]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],1] at tag 15
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],2]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],2] at tag 15
[cn1:14209] [[15336,0],0] orte:daemon:send_relay sending relay msg to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],4] at tag 15
[cn1:14209] [[15336,0],0] rml_send_buffer to peer [[15336,0],0] at tag 1
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],1] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],2] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],4] at tag 15
[cn1:14209] [[15336,0],0] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn1:14209] [[15336,0],0] rml_send_msg to peer [[15336,0],0] at tag 1
[cn1:14209] [[15336,0],0] rml_send_iovec_to_self at tag 1
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],1]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],1]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],2]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],2]
[cn1:14209] [[15336,0],0] oob:base:send to target [[15336,0],4]
[cn1:14209] [[15336,0],0] oob:base:send known transport for peer [[15336,0],4]
[cn1:14209] [[15336,0],0] oob:tcp:send_nb to peer [[15336,0],4]:15
[cn1:14209] [[15336,0],0] tcp:send_nb to peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],4]
[cn1:14209] [[15336,0],0] rml_send_to_self callback executing for tag 1
[cn1:14209] [[15336,0],0] message received 8 bytes from [[15336,0],0] for tag 1
[cn1:14209] [[15336,0],0] orted:comm:process_commands() Processing Command: ORTE_DAEMON_EXIT_CMD
[cn1:14209] [[15336,0],0] orted_cmd: received exit cmd
[cn1:14209] sess_dir_finalize: proc session dir does not exist
[cn1:14209] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn1:14209] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn1:14209] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn1:14209] [[15336,0],0] num routes 3
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],1]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],1]) --> [[15336,0],1]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],1] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],1]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],2]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],2]) --> [[15336,0],2]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],2] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],2]
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],4]:15
[cn1:14209] [[15336,0],0] routed_binomial_get([[15336,0],4]) --> [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_nb: already connected to [[15336,0],4] - queueing for send
[cn1:14209] [[15336,0],0]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],1]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],1]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],1] OF 8 BYTES ON SOCKET 14
[cn1:14209] [[15336,0],0]-[[15336,0],1] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],4]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],4]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],4] OF 8 BYTES ON SOCKET 15
[cn1:14209] [[15336,0],0]-[[15336,0],4] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] [[15336,0],0] tcp:send_handler called to send to peer [[15336,0],2]
[cn1:14209] [[15336,0],0] tcp:send_handler SENDING TO [[15336,0],2]
[cn1:14209] [[15336,0],0] MESSAGE SEND COMPLETE TO [[15336,0],2] OF 8 BYTES ON SOCKET 17
[cn1:14209] [[15336,0],0]-[[15336,0],2] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn1:14209] sess_dir_finalize: proc session dir does not exist
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate new recv msg
[cn5:14171] [[15336,0],4]:tcp:recv:handler read hdr
[cn5:14171] [[15336,0],4]:tcp:recv:handler allocate data region of size 8
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate new recv msg
[cn3:21416] [[15336,0],2]:tcp:recv:handler read hdr
[cn3:21416] [[15336,0],2]:tcp:recv:handler allocate data region of size 8
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate data region of size 8
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 5
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],1] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],1]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 2 state ABORTED BY SIGNAL exit_code 139
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn2:01042] [[15336,0],1] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 8 BYTES FOR DEST [[15336,0],1] TAG 15
[cn2:01042] [[15336,0],1] DELIVERING TO RML
[cn2:01042] [[15336,0],1] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn2:01042] [[15336,0],1] message received 8 bytes from [[15336,0],0] for tag 15
[cn2:01042] [[15336,0],1] grpcomm:xcast:recv: with 8 bytes
[cn2:01042] [[15336,0],1] orte:daemon:send_relay sending relay msg to [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],3] at tag 15
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],1] at tag 1
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],3] at tag 15
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],1] at tag 1
[cn2:01042] [[15336,0],1] rml_send_iovec_to_self at tag 1
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],3]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],3]:15
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],3]
[cn2:01042] [[15336,0],1] rml_send_to_self callback executing for tag 1
[cn2:01042] [[15336,0],1] message received 8 bytes from [[15336,0],1] for tag 1
[cn2:01042] [[15336,0],1] orted:comm:process_commands() Processing Command: ORTE_DAEMON_EXIT_CMD
[cn2:01042] [[15336,0],1] orted_cmd: received exit cmd
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 5
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],3] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],3]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 6 state ABORTED BY SIGNAL exit_code 139
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 5
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],2] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],2]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 4 state ABORTED BY SIGNAL exit_code 139
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] [[15336,0],2]:tcp:recv:handler called for peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:tcp:recv:handler CONNECTED
[cn3:21416] [[15336,0],2] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 8 BYTES FOR DEST [[15336,0],2] TAG 15
[cn3:21416] [[15336,0],2] DELIVERING TO RML
[cn3:21416] [[15336,0],2] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn3:21416] [[15336,0],2] message received 8 bytes from [[15336,0],0] for tag 15
[cn3:21416] [[15336,0],2] grpcomm:xcast:recv: with 8 bytes
[cn3:21416] [[15336,0],2] orte:daemon:send_relay - recipient list is empty!
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],2] at tag 1
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],2] at tag 1
[cn3:21416] [[15336,0],2] rml_send_iovec_to_self at tag 1
[cn3:21416] [[15336,0],2] rml_send_to_self callback executing for tag 1
[cn3:21416] [[15336,0],2] message received 8 bytes from [[15336,0],2] for tag 1
[cn3:21416] [[15336,0],2] orted:comm:process_commands() Processing Command: ORTE_DAEMON_EXIT_CMD
[cn3:21416] [[15336,0],2] orted_cmd: received exit cmd
[cn2:01042] sess_dir_finalize: proc session dir does not exist
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn2:01042] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn2:01042] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn2:01042] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn2:01042] [[15336,0],1] num routes 1
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],3]:15
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],3]) --> [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],3] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],3]
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],0] at tag 5
[cn4:17828] [[15336,0],3] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn4:17828] [[15336,0],3] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],3]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],3]
[cn4:17828] [[15336,0],3] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn4:17828] [[15336,0],3] tcp:send_nb to peer [[15336,0],0]
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],3] OF 8 BYTES ON SOCKET 10
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn4:17828] [[15336,0],3] routing direct to the HNP
[cn4:17828] [[15336,0],3] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]-[[15336,0],3] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] sess_dir_finalize: proc session dir does not exist
[cn4:17828] [[15336,0],3] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn4:17828] [[15336,0],3] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1] num routes 1
[cn2:01042] [[15336,0],1] rml_send_msg to peer [[15336,0],0] at tag 5
[cn2:01042] [[15336,0],1] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn2:01042] [[15336,0],1] oob:base:send to target [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:base:send known transport for peer [[15336,0],0]
[cn2:01042] [[15336,0],1] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn2:01042] [[15336,0],1] tcp:send_nb to peer [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn2:01042] [[15336,0],1] routing direct to the HNP
[cn2:01042] [[15336,0],1] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn2:01042] [[15336,0],1]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler called to send to peer [[15336,0],0]
[cn2:01042] [[15336,0],1] tcp:send_handler SENDING TO [[15336,0],0]
[cn2:01042] [[15336,0],1] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn2:01042] [[15336,0],1]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate new recv msg
[cn4:17828] [[15336,0],3]:tcp:recv:handler read hdr
[cn4:17828] [[15336,0],3]:tcp:recv:handler allocate data region of size 8
[cn4:17828] [[15336,0],3]:tcp:recv:handler called for peer [[15336,0],1]
[cn4:17828] [[15336,0],3]:tcp:recv:handler CONNECTED
[cn4:17828] [[15336,0],3] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 8 BYTES FOR DEST [[15336,0],3] TAG 15
[cn4:17828] [[15336,0],3] DELIVERING TO RML
[cn4:17828] [[15336,0],3] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn4:17828] [[15336,0],3] message received 8 bytes from [[15336,0],1] for tag 15
[cn4:17828] [[15336,0],3] grpcomm:xcast:recv: with 8 bytes
[cn4:17828] [[15336,0],3] orte:daemon:send_relay - recipient list is empty!
[cn4:17828] [[15336,0],3] rml_send_buffer to peer [[15336,0],3] at tag 1
[cn4:17828] [[15336,0],3] rml_send_msg to peer [[15336,0],3] at tag 1
[cn4:17828] [[15336,0],3] rml_send_iovec_to_self at tag 1
[cn4:17828] [[15336,0],3] rml_send_to_self callback executing for tag 1
[cn4:17828] [[15336,0],3] message received 8 bytes from [[15336,0],3] for tag 1
[cn4:17828] [[15336,0],3] orted:comm:process_commands() Processing Command: ORTE_DAEMON_EXIT_CMD
[cn4:17828] [[15336,0],3] orted_cmd: received exit cmd
[cn4:17828] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],3] (ORIGIN [[15336,0],3]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],3] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],3]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 7 state ABORTED BY SIGNAL exit_code 139
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 5
[cn4:17828] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn4:17828] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn4:17828] [[15336,0],3] num routes 0
[cn4:17828] [[15336,0],3] orted_cmd: all routes and children gone - exiting
[cn4:17828] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn4:17828] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn4:17828] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn4:17828] [[15336,0],3] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 21
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 62
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]-[[15336,0],3] mca_oob_tcp_msg_recv: peer closed connection
[cn1:14209] [[15336,0],0] tcp_peer_close for [[15336,0],3] sd 16 state CONNECTED
[cn1:14209] [[15336,0],0] tcp:lost connection called for peer [[15336,0],3]
[cn1:14209] [[15336,0],0] route to [[15336,0],3] lost
[cn1:14209] [[15336,0],0] route to [[15336,0],3] lost
[cn1:14209] [[15336,0],0] num routes 3
[cn1:14209] [[15336,0],0] num routes 3
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 62 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 62 bytes from [[15336,0],4] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],4]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 8 state ABORTED BY SIGNAL exit_code 139
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn2:01042] [[15336,0],1]:tcp:recv:handler called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1]:tcp:recv:handler CONNECTED
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn4:17828] [[15336,0],3] grpcomm:base:receive stop comm
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn4:17828] [[15336,0],3] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 30
[cn4:17828] [[15336,0],3] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 15
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 32
[cn4:17828] mca: base: close: component bad closed
[cn4:17828] mca: base: close: unloading component bad
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn4:17828] [[15336,0],3] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 3
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 62 BYTES ON SOCKET 9
[cn4:17828] [[15336,0],3] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 45
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn4:17828] [[15336,0],3] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 46
[cn5:14171] [[15336,0],4]:tcp:recv:handler called for peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:tcp:recv:handler CONNECTED
[cn4:17828] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn5:14171] [[15336,0],4] RECVD COMPLETE MESSAGE FROM [[15336,0],0] (ORIGIN [[15336,0],0]) OF 8 BYTES FOR DEST [[15336,0],4] TAG 15
[cn4:17828] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn4:17828] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn5:14171] [[15336,0],4] DELIVERING TO RML
[cn5:14171] [[15336,0],4] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn5:14171] [[15336,0],4] message received 8 bytes from [[15336,0],0] for tag 15
[cn4:17828] mca: base: close: unloading component binomial
[cn5:14171] [[15336,0],4] grpcomm:xcast:recv: with 8 bytes
[cn5:14171] [[15336,0],4] orte:daemon:send_relay - recipient list is empty!
[cn4:17828] mca: base: close: component oob closed
[cn4:17828] mca: base: close: unloading component oob
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],4] at tag 1
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],4] at tag 1
[cn4:17828] [[15336,0],3] TCP SHUTDOWN
[cn5:14171] [[15336,0],4] rml_send_iovec_to_self at tag 1
[cn5:14171] [[15336,0],4] rml_send_to_self callback executing for tag 1
[cn5:14171] [[15336,0],4] message received 8 bytes from [[15336,0],4] for tag 1
[cn4:17828] [[15336,0],3] RELEASING PEER OBJ [[15336,0],0]
[cn5:14171] [[15336,0],4] orted:comm:process_commands() Processing Command: ORTE_DAEMON_EXIT_CMD
[cn5:14171] [[15336,0],4] orted_cmd: received exit cmd
[cn4:17828] [[15336,0],3] CLOSING SOCKET 9
[cn4:17828] [[15336,0],3] RELEASING PEER OBJ [[15336,0],1]
[cn4:17828] [[15336,0],3] CLOSING SOCKET 10
[cn4:17828] [[15336,0],3] RELEASING PEER OBJ [[15336,0],2]
[cn4:17828] [[15336,0],3] RELEASING PEER OBJ [[15336,0],4]
[cn4:17828] [[15336,0],3] RELEASING PEER OBJ [[15336,1],6]
[cn4:17828] [[15336,0],3] CLOSING SOCKET 13
[cn4:17828] [[15336,0],3] RELEASING PEER OBJ [[15336,1],7]
[cn4:17828] [[15336,0],3] CLOSING SOCKET 14
[cn4:17828] mca: base: close: component tcp closed
[cn4:17828] mca: base: close: unloading component tcp
[cn2:01042] [[15336,0],1]:tcp:recv:handler allocate new recv msg
[cn2:01042] [[15336,0],1]:tcp:recv:handler read hdr
[cn2:01042] [[15336,0],1]-[[15336,0],3] mca_oob_tcp_msg_recv: peer closed connection
[cn2:01042] [[15336,0],1] tcp_peer_close for [[15336,0],3] sd 10 state CONNECTED
[cn2:01042] [[15336,0],1] tcp:lost connection called for peer [[15336,0],3]
[cn2:01042] [[15336,0],1] route to [[15336,0],3] lost
[cn2:01042] [[15336,0],1] routed_binomial: removing route to child daemon [[15336,0],3]
[cn2:01042] [[15336,0],1] num routes 0
[cn2:01042] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn2:01042] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn2:01042] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn2:01042] [[15336,0],1] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 21
[cn2:01042] [[15336,0],1] grpcomm:base:receive stop comm
[cn2:01042] [[15336,0],1] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 30
[cn2:01042] [[15336,0],1] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 15
[cn2:01042] [[15336,0],1] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 32
[cn2:01042] mca: base: close: component bad closed
[cn2:01042] mca: base: close: unloading component bad
[cn2:01042] [[15336,0],1] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 3
[cn4:17828] mca: base: close: unloading component hash
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],1] (ORIGIN [[15336,0],1]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],1] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],1]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 3 state NORMALLY TERMINATED exit_code 11
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]-[[15336,0],1] mca_oob_tcp_msg_recv: peer closed connection
[cn1:14209] [[15336,0],0] tcp_peer_close for [[15336,0],1] sd 14 state CONNECTED
[cn1:14209] [[15336,0],0] tcp:lost connection called for peer [[15336,0],1]
[cn1:14209] [[15336,0],0] route to [[15336,0],1] lost
[cn1:14209] [[15336,0],0] routed_binomial: removing route to child daemon [[15336,0],1]
[cn1:14209] [[15336,0],0] route to [[15336,0],1] lost
[cn1:14209] [[15336,0],0] num routes 2
[cn1:14209] [[15336,0],0] num routes 2
[cn2:01042] [[15336,0],1] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 45
[cn2:01042] [[15336,0],1] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 46
[cn2:01042] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn2:01042] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn2:01042] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn2:01042] mca: base: close: unloading component binomial
[cn2:01042] mca: base: close: component oob closed
[cn2:01042] mca: base: close: unloading component oob
[cn2:01042] [[15336,0],1] TCP SHUTDOWN
[cn2:01042] [[15336,0],1] RELEASING PEER OBJ [[15336,0],0]
[cn2:01042] [[15336,0],1] CLOSING SOCKET 9
[cn2:01042] [[15336,0],1] RELEASING PEER OBJ [[15336,0],2]
[cn4:17828] sess_dir_cleanup: job session dir does not exist
[cn2:01042] [[15336,0],1] RELEASING PEER OBJ [[15336,0],3]
[cn2:01042] [[15336,0],1] CLOSING SOCKET 10
[cn2:01042] [[15336,0],1] RELEASING PEER OBJ [[15336,0],4]
[cn2:01042] [[15336,0],1] RELEASING PEER OBJ [[15336,1],2]
[cn2:01042] [[15336,0],1] CLOSING SOCKET 13
[cn2:01042] [[15336,0],1] RELEASING PEER OBJ [[15336,1],3]
[cn2:01042] [[15336,0],1] CLOSING SOCKET 14
[cn2:01042] mca: base: close: component tcp closed
[cn2:01042] mca: base: close: unloading component tcp
[cn4:17828] mca: base: close: component slurm closed
[cn4:17828] mca: base: close: unloading component slurm
[cn2:01042] mca: base: close: unloading component hash
[cn3:21416] sess_dir_finalize: proc session dir does not exist
[cn3:21416] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn3:21416] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn3:21416] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn3:21416] [[15336,0],2] num routes 0
[cn2:01042] sess_dir_cleanup: job session dir does not exist
[cn3:21416] sess_dir_finalize: proc session dir does not exist
[cn3:21416] [[15336,0],2] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn3:21416] [[15336,0],2] num routes 0
[cn3:21416] [[15336,0],2] rml_send_msg to peer [[15336,0],0] at tag 5
[cn3:21416] [[15336,0],2] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn3:21416] [[15336,0],2] oob:base:send to target [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:base:send known transport for peer [[15336,0],0]
[cn3:21416] [[15336,0],2] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn3:21416] [[15336,0],2] tcp:send_nb to peer [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn3:21416] [[15336,0],2] routing direct to the HNP
[cn2:01042] mca: base: close: component slurm closed
[cn2:01042] mca: base: close: unloading component slurm
[cn3:21416] [[15336,0],2] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn3:21416] [[15336,0],2]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler called to send to peer [[15336,0],0]
[cn3:21416] [[15336,0],2] tcp:send_handler SENDING TO [[15336,0],0]
[cn3:21416] [[15336,0],2] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn3:21416] [[15336,0],2]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn3:21416] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn3:21416] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn3:21416] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn3:21416] [[15336,0],2] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 21
[cn3:21416] [[15336,0],2] grpcomm:base:receive stop comm
[cn3:21416] [[15336,0],2] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 30
[cn3:21416] [[15336,0],2] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 15
[cn3:21416] [[15336,0],2] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 32
[cn3:21416] mca: base: close: component bad closed
[cn3:21416] mca: base: close: unloading component bad
[cn3:21416] [[15336,0],2] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 3
[cn3:21416] [[15336,0],2] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 45
[cn3:21416] [[15336,0],2] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 46
[cn3:21416] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn3:21416] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn3:21416] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn3:21416] mca: base: close: unloading component binomial
[cn3:21416] mca: base: close: component oob closed
[cn3:21416] mca: base: close: unloading component oob
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],2] (ORIGIN [[15336,0],2]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],2] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],2]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 5 state NORMALLY TERMINATED exit_code 11
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]-[[15336,0],2] mca_oob_tcp_msg_recv: peer closed connection
[cn1:14209] [[15336,0],0] tcp_peer_close for [[15336,0],2] sd 17 state CONNECTED
[cn1:14209] [[15336,0],0] tcp:lost connection called for peer [[15336,0],2]
[cn1:14209] [[15336,0],0] route to [[15336,0],2] lost
[cn1:14209] [[15336,0],0] routed_binomial: removing route to child daemon [[15336,0],2]
[cn1:14209] [[15336,0],0] route to [[15336,0],2] lost
[cn1:14209] [[15336,0],0] num routes 1
[cn1:14209] [[15336,0],0] num routes 1
exiting with status 0
[cn3:21416] [[15336,0],2] TCP SHUTDOWN
[cn3:21416] [[15336,0],2] RELEASING PEER OBJ [[15336,0],0]
[cn3:21416] [[15336,0],2] CLOSING SOCKET 9
[cn3:21416] [[15336,0],2] RELEASING PEER OBJ [[15336,0],1]
[cn3:21416] [[15336,0],2] RELEASING PEER OBJ [[15336,0],3]
[cn3:21416] [[15336,0],2] RELEASING PEER OBJ [[15336,0],4]
[cn3:21416] [[15336,0],2] RELEASING PEER OBJ [[15336,1],4]
[cn3:21416] [[15336,0],2] CLOSING SOCKET 12
[cn3:21416] [[15336,0],2] RELEASING PEER OBJ [[15336,1],5]
[cn3:21416] [[15336,0],2] CLOSING SOCKET 13
[cn3:21416] mca: base: close: component tcp closed
[cn3:21416] mca: base: close: unloading component tcp
[cn3:21416] mca: base: close: unloading component hash
exiting with status 0
[cn3:21416] sess_dir_cleanup: job session dir does not exist
[cn3:21416] mca: base: close: component slurm closed
[cn3:21416] mca: base: close: unloading component slurm
[cn5:14171] sess_dir_finalize: proc session dir does not exist
[cn5:14171] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn5:14171] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn5:14171] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn5:14171] [[15336,0],4] num routes 0
[cn5:14171] sess_dir_finalize: proc session dir does not exist
[cn5:14171] [[15336,0],4] rml_send_buffer to peer [[15336,0],0] at tag 5
[cn5:14171] [[15336,0],4] num routes 0
[cn5:14171] [[15336,0],4] rml_send_msg to peer [[15336,0],0] at tag 5
[cn5:14171] [[15336,0],4] OOB_SEND: /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/rml/oob/rml_oob_send.c:199
[cn5:14171] [[15336,0],4] oob:base:send to target [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:base:send known transport for peer [[15336,0],0]
[cn5:14171] [[15336,0],4] oob:tcp:send_nb to peer [[15336,0],0]:5
[cn5:14171] [[15336,0],4] tcp:send_nb to peer [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:508] post send to [[15336,0],0]
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:442] processing send to peer [[15336,0],0]:5
[cn5:14171] [[15336,0],4] routing direct to the HNP
[cn5:14171] [[15336,0],4] routed_binomial_get([[15336,0],0]) --> [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_nb: already connected to [[15336,0],0] - queueing for send
[cn5:14171] [[15336,0],4]:[/home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp.c:469] queue send to [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler called to send to peer [[15336,0],0]
[cn5:14171] [[15336,0],4] tcp:send_handler SENDING TO [[15336,0],0]
[cn5:14171] [[15336,0],4] MESSAGE SEND COMPLETE TO [[15336,0],0] OF 73 BYTES ON SOCKET 9
[cn5:14171] [[15336,0],4]-[[15336,0],0] Send message complete at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:224
[cn5:14171] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn5:14171] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn5:14171] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn5:14171] [[15336,0],4] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 21
[cn5:14171] [[15336,0],4] grpcomm:base:receive stop comm
[cn5:14171] [[15336,0],4] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 30
[cn5:14171] [[15336,0],4] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 15
[cn5:14171] [[15336,0],4] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 32
[cn5:14171] mca: base: close: component bad closed
[cn5:14171] mca: base: close: unloading component bad
[cn5:14171] [[15336,0],4] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 3
[cn5:14171] [[15336,0],4] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 45
[cn5:14171] [[15336,0],4] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 46
[cn5:14171] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate data region of size 73
[cn1:14209] [[15336,0],0] RECVD COMPLETE MESSAGE FROM [[15336,0],4] (ORIGIN [[15336,0],4]) OF 73 BYTES FOR DEST [[15336,0],0] TAG 5
[cn1:14209] [[15336,0],0] DELIVERING TO RML
[cn1:14209] [[15336,0],0] Message posted at /home/research/artpol/ompi_dev//ompi-trunk_r31907/orte/mca/oob/tcp/oob_tcp_sendrecv.c:536
[cn1:14209] [[15336,0],0] message received 73 bytes from [[15336,0],4] for tag 5
[cn1:14209] [[15336,0],0] plm:base:receive processing msg
[cn1:14209] [[15336,0],0] plm:base:receive update proc state command from [[15336,0],4]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for job [15336,1]
[cn1:14209] [[15336,0],0] plm:base:receive got update_proc_state for vpid 9 state NORMALLY TERMINATED exit_code 11
[cn1:14209] [[15336,0],0] plm:base:receive done processing commands
[cn1:14209] [[15336,0],0]:tcp:recv:handler called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0]:tcp:recv:handler CONNECTED
[cn1:14209] [[15336,0],0]:tcp:recv:handler allocate new recv msg
[cn1:14209] [[15336,0],0]:tcp:recv:handler read hdr
[cn1:14209] [[15336,0],0]-[[15336,0],4] mca_oob_tcp_msg_recv: peer closed connection
[cn1:14209] [[15336,0],0] tcp_peer_close for [[15336,0],4] sd 15 state CONNECTED
[cn5:14171] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn5:14171] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn1:14209] [[15336,0],0] tcp:lost connection called for peer [[15336,0],4]
[cn1:14209] [[15336,0],0] route to [[15336,0],4] lost
[cn5:14171] mca: base: close: unloading component binomial
[cn1:14209] [[15336,0],0] routed_binomial: removing route to child daemon [[15336,0],4]
exiting with status 0
[cn5:14171] mca: base: close: component oob closed
[cn5:14171] mca: base: close: unloading component oob
[cn5:14171] [[15336,0],4] TCP SHUTDOWN
[cn1:14209] [[15336,0],0] route to [[15336,0],4] lost
[cn1:14209] [[15336,0],0] num routes 0
[cn5:14171] [[15336,0],4] RELEASING PEER OBJ [[15336,0],0]
[cn5:14171] [[15336,0],4] CLOSING SOCKET 9
[cn5:14171] [[15336,0],4] RELEASING PEER OBJ [[15336,0],1]
[cn5:14171] [[15336,0],4] RELEASING PEER OBJ [[15336,0],2]
[cn5:14171] [[15336,0],4] RELEASING PEER OBJ [[15336,0],3]
[cn5:14171] [[15336,0],4] RELEASING PEER OBJ [[15336,1],8]
[cn5:14171] [[15336,0],4] CLOSING SOCKET 12
[cn5:14171] [[15336,0],4] RELEASING PEER OBJ [[15336,1],9]
[cn5:14171] [[15336,0],4] CLOSING SOCKET 13
[cn5:14171] mca: base: close: component tcp closed
[cn5:14171] mca: base: close: unloading component tcp
[cn5:14171] mca: base: close: unloading component hash
[cn5:14171] sess_dir_cleanup: job session dir does not exist
[cn5:14171] mca: base: close: component slurm closed
[cn5:14171] mca: base: close: unloading component slurm
exiting with status 0
[cn1:14209] Waiting for thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn1:14209] Thread obtained /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:118
[cn1:14209] Releasing thread /home/research/artpol/ompi_dev//ompi-trunk_r31907/opal/class/opal_ring_buffer.c:135
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 34
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 45
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 46
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 21
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 22
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 2
[cn1:14209] mca: base: close: unloading component lama
[cn1:14209] mca: base: close: component mindist closed
[cn1:14209] mca: base: close: unloading component mindist
[cn1:14209] mca: base: close: component ppr closed
[cn1:14209] mca: base: close: unloading component ppr
[cn1:14209] mca: base: close: component rank_file closed
[cn1:14209] mca: base: close: unloading component rank_file
[cn1:14209] mca: base: close: component resilient closed
[cn1:14209] mca: base: close: unloading component resilient
[cn1:14209] mca: base: close: component round_robin closed
[cn1:14209] mca: base: close: unloading component round_robin
[cn1:14209] mca: base: close: component seq closed
[cn1:14209] mca: base: close: unloading component seq
[cn1:14209] mca: base: close: component staged closed
[cn1:14209] mca: base: close: unloading component staged
[cn1:14209] mca: base: close: component slurm closed
[cn1:14209] mca: base: close: unloading component slurm
[cn1:14209] [[15336,0],0] grpcomm:base:receive stop comm
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 30
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 15
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 32
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 33
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 48
[cn1:14209] mca: base: close: component bad closed
[cn1:14209] mca: base: close: unloading component bad
[cn1:14209] mca: base: close: unloading component hash
[cn1:14209] mca: base: close: unloading component binomial
[cn1:14209] [[15336,0],0] plm:base:receive stop comm
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 5
[cn1:14209] [[15336,0],0] rml_recv_cancel for peer [[WILDCARD],WILDCARD] tag 10
[cn1:14209] mca: base: close: component slurm closed
[cn1:14209] mca: base: close: unloading component slurm
[cn1:14209] mca: base: close: component oob closed
[cn1:14209] mca: base: close: unloading component oob
[cn1:14209] [[15336,0],0] TCP SHUTDOWN
[cn1:14209] [[15336,0],0] RELEASING PEER OBJ [[15336,0],1]
[cn1:14209] [[15336,0],0] CLOSING SOCKET 14
[cn1:14209] [[15336,0],0] RELEASING PEER OBJ [[15336,0],4]
[cn1:14209] [[15336,0],0] CLOSING SOCKET 15
[cn1:14209] [[15336,0],0] RELEASING PEER OBJ [[15336,0],3]
[cn1:14209] [[15336,0],0] CLOSING SOCKET 16
[cn1:14209] [[15336,0],0] RELEASING PEER OBJ [[15336,0],2]
[cn1:14209] [[15336,0],0] CLOSING SOCKET 17
[cn1:14209] [[15336,0],0] RELEASING PEER OBJ [[15336,1],0]
[cn1:14209] [[15336,0],0] CLOSING SOCKET 20
[cn1:14209] [[15336,0],0] RELEASING PEER OBJ [[15336,1],1]
[cn1:14209] [[15336,0],0] CLOSING SOCKET 0
[cn1:14209] mca: base: close: component tcp closed
[cn1:14209] mca: base: close: unloading component tcp
[cn1:14209] sess_dir_cleanup: job session dir does not exist
[cn1:14209] mca: base: close: component hnp closed
[cn1:14209] mca: base: close: unloading component hnp
exiting with status 139
