Re: [OMPI devel] hwloc out-of-order topology discovery with SLURM 14.11.0 and openmpi 1.6

[Pim Schellart]

Dear Gilles et al.,

we tested with openmpi compiled from source (version 1.8.3) both with:

./configure --prefix=/usr/local/openmpi --disable-silent-rules 
--with-libltdl=external --with-devel-headers --with-slurm 
--enable-heterogeneous --disable-vt --sysconfdir=/etc/openmpi

and

./configure --prefix=/usr/local/openmpi --with-hwloc=/usr 
--disable-silent-rules --with-libltdl=external --with-devel-headers 
--with-slurm --enable-heterogeneous --disable-vt --sysconfdir=/etc/openmpi

(e.g. with embedded and external hwloc) and the issue remains the same. 
Meanwhile we have found another interesting detail. A job is started consisting 
of four tasks split over two nodes. If this is the only job running on those 
nodes the out-of-order warnings do not appear. However, if multiple jobs are 
running the warnings do appear but only for the jobs that are started later. We 
suspect that this is because for the first started job the CPU cores assigned 
are 0 and 1 whereas they are different for the later started jobs. I attached 
the output (including lstopo —of xml output (called for each task)) for both 
the working and broken case again.

Kind regards,

Pim Schellart


<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE topology SYSTEM "hwloc.dtd">
<topology>
  <object type="Machine" os_index="0" cpuset="0x00000004" 
complete_cpuset="0x0000ffff" online_cpuset="0x0000ffff" 
allowed_cpuset="0x00000004" nodeset="0x00000003" complete_nodeset="0x00000003" 
allowed_nodeset="0x00000003">
    <page_type size="4096" count="0"/>
    <page_type size="2097152" count="0"/>
    <info name="DMIProductName" value="PowerEdge R720"/>
    <info name="DMIProductVersion" value=""/>
    <info name="DMIBoardVendor" value="Dell Inc."/>
    <info name="DMIBoardName" value="046V88"/>
    <info name="DMIBoardVersion" value="A00"/>
    <info name="DMIBoardAssetTag" value=""/>
    <info name="DMIChassisVendor" value="Dell Inc."/>
    <info name="DMIChassisType" value="23"/>
    <info name="DMIChassisVersion" value=""/>
    <info name="DMIChassisAssetTag" value=""/>
    <info name="DMIBIOSVendor" value="Dell Inc."/>
    <info name="DMIBIOSVersion" value="1.2.6"/>
    <info name="DMIBIOSDate" value="05/10/2012"/>
    <info name="DMISysVendor" value="Dell Inc."/>
    <info name="Backend" value="Linux"/>
    <info name="LinuxCgroup" value="/slurm/uid_11922/job_106635/step_0"/>
    <info name="OSName" value="Linux"/>
    <info name="OSRelease" value="3.13.0-40-generic"/>
    <info name="OSVersion" value="#69-Ubuntu SMP Thu Nov 13 17:53:56 UTC 2014"/>
    <info name="HostName" value="coma04"/>
    <info name="Architecture" value="x86_64"/>
    <distances nbobjs="2" relative_depth="1" latency_base="10.000000">
      <latency value="1.000000"/>
      <latency value="2.000000"/>
      <latency value="2.000000"/>
      <latency value="1.000000"/>
    </distances>
    <object type="NUMANode" os_index="0" cpuset="0x00000004" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" local_memory="67514843136">
      <page_type size="4096" count="16483116"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="0" cpuset="0x00000004" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000004" complete_cpuset="0x00005555" 
online_cpuset="0x00005555" allowed_cpuset="0x00000004" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000004" complete_cpuset="0x00000004" 
online_cpuset="0x00000004" allowed_cpuset="0x00000004" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="1" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="2" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
    </object>
    <object type="NUMANode" os_index="1" cpuset="0x0" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" allowed_cpuset="0x0" 
nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" 
local_memory="67643228160">
      <page_type size="4096" count="16514460"/>
      <page_type size="2097152" count="0"/>
    </object>
    <object type="Bridge" os_index="0" bridge_type="0-1" depth="0" 
bridge_pci="0000:[00-0d]">
      <object type="Bridge" os_index="16" name="Intel Corporation Xeon E5/Core 
i7 IIO PCI Express Root Port 1a" bridge_type="1-1" depth="1" 
bridge_pci="0000:[02-02]" pci_busid="0000:00:01.0" pci_type="0604 [8086:3c02] 
[0000:0000] 07" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root Port 
1a"/>
        <object type="PCIDev" os_index="8192" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Broadcom Corporation"/>
          <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
          <object type="OSDev" name="em3" osdev_type="2">
            <info name="Address" value="d4:ae:52:ad:88:12"/>
          </object>
        </object>
        <object type="PCIDev" os_index="8193" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Broadcom Corporation"/>
          <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
          <object type="OSDev" name="em4" osdev_type="2">
            <info name="Address" value="d4:ae:52:ad:88:13"/>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="17" name="Intel Corporation Xeon E5/Core 
i7 IIO PCI Express Root Port 1b" bridge_type="1-1" depth="1" 
bridge_pci="0000:[01-01]" pci_busid="0000:00:01.1" pci_type="0604 [8086:3c03] 
[0000:0000] 07" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root Port 
1b"/>
        <object type="PCIDev" os_index="4096" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Broadcom Corporation"/>
          <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
          <object type="OSDev" name="em1" osdev_type="2">
            <info name="Address" value="d4:ae:52:ad:88:10"/>
          </object>
        </object>
        <object type="PCIDev" os_index="4097" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Broadcom Corporation"/>
          <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
          <object type="OSDev" name="em2" osdev_type="2">
            <info name="Address" value="d4:ae:52:ad:88:11"/>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="34" name="Intel Corporation Xeon E5/Core 
i7 IIO PCI Express Root Port 2c" bridge_type="1-1" depth="1" 
bridge_pci="0000:[03-03]" pci_busid="0000:00:02.2" pci_type="0604 [8086:3c06] 
[0000:0000] 07" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root Port 
2c"/>
        <object type="PCIDev" os_index="12288" name="LSI Logic / Symbios Logic 
MegaRAID SAS 2208 [Thunderbolt]" pci_busid="0000:03:00.0" pci_type="0104 
[1000:005b] [1028:1f38] 01" pci_link_speed="0.000000">
          <info name="PCIVendor" value="LSI Logic / Symbios Logic"/>
          <info name="PCIDevice" value="MegaRAID SAS 2208 [Thunderbolt]"/>
          <object type="OSDev" name="sda" osdev_type="0"/>
        </object>
      </object>
      <object type="Bridge" os_index="48" name="Intel Corporation Xeon E5/Core 
i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="1" bridge_pci="0000:[05-05]" pci_busid="0000:00:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root Port 
3a in PCI Express Mode"/>
        <object type="PCIDev" os_index="20480" name="NVIDIA Corporation GF110GL 
[Tesla M2090]" pci_busid="0000:05:00.0" pci_type="0302 [10de:1091] [10de:0887] 
a1" pci_link_speed="0.000000">
          <info name="PCIVendor" value="NVIDIA Corporation"/>
          <info name="PCIDevice" value="GF110GL [Tesla M2090]"/>
          <object type="OSDev" name="card0" osdev_type="1"/>
        </object>
      </object>
      <object type="Bridge" os_index="455" name="Intel Corporation C600/X79 
series chipset PCI Express Root Port 8" bridge_type="1-1" depth="1" 
bridge_pci="0000:[08-0c]" pci_busid="0000:00:1c.7" pci_type="0604 [8086:1d1e] 
[0000:0000] b5" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="C600/X79 series chipset PCI Express Root 
Port 8"/>
        <object type="Bridge" os_index="32768" name="Renesas Technology Corp. 
SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="2" bridge_pci="0000:[09-0c]" 
pci_busid="0000:08:00.0" pci_type="0604 [1912:0013] [0000:0000] 00" 
pci_link_speed="0.000000">
          <info name="PCIVendor" value="Renesas Technology Corp."/>
          <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
          <object type="Bridge" os_index="36864" name="Renesas Technology Corp. 
SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="3" bridge_pci="0000:[0a-0b]" 
pci_busid="0000:09:00.0" pci_type="0604 [1912:0013] [0000:0000] 00" 
pci_link_speed="0.000000">
            <info name="PCIVendor" value="Renesas Technology Corp."/>
            <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
            <object type="Bridge" os_index="40960" name="Renesas Technology 
Corp. SH7757 PCIe-PCI Bridge [PPB]" bridge_type="1-1" depth="4" 
bridge_pci="0000:[0b-0b]" pci_busid="0000:0a:00.0" pci_type="0604 [1912:0012] 
[0000:0000] 00" pci_link_speed="0.000000">
              <info name="PCIVendor" value="Renesas Technology Corp."/>
              <info name="PCIDevice" value="SH7757 PCIe-PCI Bridge [PPB]"/>
              <object type="PCIDev" os_index="45056" name="Matrox Electronics 
Systems Ltd. G200eR2" pci_busid="0000:0b:00.0" pci_type="0300 [102b:0534] 
[1028:048c] 00" pci_link_speed="0.000000">
                <info name="PCIVendor" value="Matrox Electronics Systems Ltd."/>
                <info name="PCIDevice" value="G200eR2"/>
              </object>
            </object>
          </object>
        </object>
      </object>
    </object>
    <object type="Bridge" os_index="2" bridge_type="0-1" depth="0" 
bridge_pci="0000:[40-44]">
      <object type="Bridge" os_index="262192" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="1" bridge_pci="0000:[43-43]" pci_busid="0000:40:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root Port 
3a in PCI Express Mode"/>
        <object type="PCIDev" os_index="274432" name="QLogic Corp. IBA7322 QDR 
InfiniBand HCA" pci_busid="0000:43:00.0" pci_type="0c06 [1077:7322] [1077:7322] 
02" pci_link_speed="0.000000">
          <info name="PCIVendor" value="QLogic Corp."/>
          <info name="PCIDevice" value="IBA7322 QDR InfiniBand HCA"/>
          <object type="OSDev" name="ib0" osdev_type="2">
            <info name="Address" 
value="80:00:00:03:fe:80:00:00:00:00:00:00:00:11:75:00:00:76:f4:80"/>
            <info name="Port" value="1"/>
          </object>
          <object type="OSDev" name="qib0" osdev_type="3">
            <info name="NodeGUID" value="0011:7500:0076:f480"/>
            <info name="SysImageGUID" value="0011:7500:0076:f480"/>
            <info name="Port1State" value="4"/>
            <info name="Port1LID" value="0x8"/>
            <info name="Port1LMC" value="0"/>
            <info name="Port1GID0" 
value="fe80:0000:0000:0000:0011:7500:0076:f480"/>
          </object>
        </object>
      </object>
    </object>
  </object>
</topology>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE topology SYSTEM "hwloc.dtd">
<topology>
  <object type="Machine" os_index="0" cpuset="0x00000058" 
complete_cpuset="0x0000ffff" online_cpuset="0x0000ffff" 
allowed_cpuset="0x00000058" nodeset="0x00000003" complete_nodeset="0x00000003" 
allowed_nodeset="0x00000003">
    <page_type size="4096" count="0"/>
    <page_type size="2097152" count="0"/>
    <info name="DMIProductName" value="PowerEdge R720"/>
    <info name="DMIProductVersion" value=""/>
    <info name="DMIBoardVendor" value="Dell Inc."/>
    <info name="DMIBoardName" value="046V88"/>
    <info name="DMIBoardVersion" value="A00"/>
    <info name="DMIBoardAssetTag" value=""/>
    <info name="DMIChassisVendor" value="Dell Inc."/>
    <info name="DMIChassisType" value="23"/>
    <info name="DMIChassisVersion" value=""/>
    <info name="DMIChassisAssetTag" value=""/>
    <info name="DMIBIOSVendor" value="Dell Inc."/>
    <info name="DMIBIOSVersion" value="1.2.6"/>
    <info name="DMIBIOSDate" value="05/10/2012"/>
    <info name="DMISysVendor" value="Dell Inc."/>
    <info name="Backend" value="Linux"/>
    <info name="LinuxCgroup" value="/slurm/uid_11922/job_106635/step_0"/>
    <info name="OSName" value="Linux"/>
    <info name="OSRelease" value="3.13.0-39-generic"/>
    <info name="OSVersion" value="#66-Ubuntu SMP Tue Oct 28 13:30:27 UTC 2014"/>
    <info name="HostName" value="coma03"/>
    <info name="Architecture" value="x86_64"/>
    <distances nbobjs="2" relative_depth="1" latency_base="10.000000">
      <latency value="1.000000"/>
      <latency value="2.000000"/>
      <latency value="2.000000"/>
      <latency value="1.000000"/>
    </distances>
    <object type="NUMANode" os_index="0" cpuset="0x00000050" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000050" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" local_memory="67514843136">
      <page_type size="4096" count="16483116"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="0" cpuset="0x00000050" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000050" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000050" complete_cpuset="0x00005555" 
online_cpuset="0x00005555" allowed_cpuset="0x00000050" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000010" complete_cpuset="0x00000010" 
online_cpuset="0x00000010" allowed_cpuset="0x00000010" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="2" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="4" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
          <object type="Cache" cpuset="0x00000040" complete_cpuset="0x00000040" 
online_cpuset="0x00000040" allowed_cpuset="0x00000040" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="3" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="6" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="0" bridge_type="0-1" depth="0" 
bridge_pci="0000:[00-0d]">
        <object type="Bridge" os_index="16" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1a" bridge_type="1-1" depth="0" 
bridge_pci="0000:[02-02]" pci_busid="0000:00:01.0" pci_type="0604 [8086:3c02] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1a"/>
          <object type="PCIDev" os_index="8192" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em3" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d7"/>
            </object>
          </object>
          <object type="PCIDev" os_index="8193" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em4" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d8"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="17" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1b" bridge_type="1-1" depth="0" 
bridge_pci="0000:[01-01]" pci_busid="0000:00:01.1" pci_type="0604 [8086:3c03] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1b"/>
          <object type="PCIDev" os_index="4096" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em1" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d5"/>
            </object>
          </object>
          <object type="PCIDev" os_index="4097" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em2" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d6"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="34" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 2c" bridge_type="1-1" depth="0" 
bridge_pci="0000:[03-03]" pci_busid="0000:00:02.2" pci_type="0604 [8086:3c06] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 2c"/>
          <object type="PCIDev" os_index="12288" name="LSI Logic / Symbios 
Logic MegaRAID SAS 2208 [Thunderbolt]" pci_busid="0000:03:00.0" pci_type="0104 
[1000:005b] [1028:1f38] 01" pci_link_speed="0.000000">
            <info name="PCIVendor" value="LSI Logic / Symbios Logic"/>
            <info name="PCIDevice" value="MegaRAID SAS 2208 [Thunderbolt]"/>
            <object type="OSDev" name="sda" osdev_type="0"/>
          </object>
        </object>
        <object type="Bridge" os_index="48" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[05-05]" pci_busid="0000:00:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="20480" name="NVIDIA Corporation 
GF110GL [Tesla M2090]" pci_busid="0000:05:00.0" pci_type="0302 [10de:1091] 
[10de:0887] a1" pci_link_speed="0.000000">
            <info name="PCIVendor" value="NVIDIA Corporation"/>
            <info name="PCIDevice" value="GF110GL [Tesla M2090]"/>
            <object type="OSDev" name="card0" osdev_type="1"/>
          </object>
        </object>
        <object type="Bridge" os_index="455" name="Intel Corporation C600/X79 
series chipset PCI Express Root Port 8" bridge_type="1-1" depth="0" 
bridge_pci="0000:[08-0c]" pci_busid="0000:00:1c.7" pci_type="0604 [8086:1d1e] 
[0000:0000] b5" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="C600/X79 series chipset PCI Express 
Root Port 8"/>
          <object type="Bridge" os_index="32768" name="Renesas Technology Corp. 
SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="1" bridge_pci="0000:[09-0c]" 
pci_busid="0000:08:00.0" pci_type="0604 [1912:0013] [0000:0000] 00" 
pci_link_speed="0.000000">
            <info name="PCIVendor" value="Renesas Technology Corp."/>
            <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
            <object type="Bridge" os_index="36864" name="Renesas Technology 
Corp. SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="2" 
bridge_pci="0000:[0a-0b]" pci_busid="0000:09:00.0" pci_type="0604 [1912:0013] 
[0000:0000] 00" pci_link_speed="0.000000">
              <info name="PCIVendor" value="Renesas Technology Corp."/>
              <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
              <object type="Bridge" os_index="40960" name="Renesas Technology 
Corp. SH7757 PCIe-PCI Bridge [PPB]" bridge_type="1-1" depth="3" 
bridge_pci="0000:[0b-0b]" pci_busid="0000:0a:00.0" pci_type="0604 [1912:0012] 
[0000:0000] 00" pci_link_speed="0.000000">
                <info name="PCIVendor" value="Renesas Technology Corp."/>
                <info name="PCIDevice" value="SH7757 PCIe-PCI Bridge [PPB]"/>
                <object type="PCIDev" os_index="45056" name="Matrox Electronics 
Systems Ltd. G200eR2" pci_busid="0000:0b:00.0" pci_type="0300 [102b:0534] 
[1028:048c] 00" pci_link_speed="0.000000">
                  <info name="PCIVendor" value="Matrox Electronics Systems 
Ltd."/>
                  <info name="PCIDevice" value="G200eR2"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
    </object>
    <object type="NUMANode" os_index="1" cpuset="0x00000008" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" local_memory="67643228160">
      <page_type size="4096" count="16514460"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="1" cpuset="0x00000008" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000008" complete_cpuset="0x0000aaaa" 
online_cpuset="0x0000aaaa" allowed_cpuset="0x00000008" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000008" complete_cpuset="0x00000008" 
online_cpuset="0x00000008" allowed_cpuset="0x00000008" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="1" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
                  <object type="PU" os_index="3" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="2" bridge_type="0-1" depth="0" 
bridge_pci="0000:[40-44]">
        <object type="Bridge" os_index="262192" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[43-43]" pci_busid="0000:40:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="274432" name="QLogic Corp. IBA7322 
QDR InfiniBand HCA" pci_busid="0000:43:00.0" pci_type="0c06 [1077:7322] 
[1077:7322] 02" pci_link_speed="0.000000">
            <info name="PCIVendor" value="QLogic Corp."/>
            <info name="PCIDevice" value="IBA7322 QDR InfiniBand HCA"/>
            <object type="OSDev" name="ib0" osdev_type="2">
              <info name="Address" 
value="80:00:00:03:fe:80:00:00:00:00:00:00:00:11:75:00:00:76:f1:86"/>
              <info name="Port" value="1"/>
            </object>
            <object type="OSDev" name="qib0" osdev_type="3">
              <info name="NodeGUID" value="0011:7500:0076:f186"/>
              <info name="SysImageGUID" value="0011:7500:0076:f186"/>
              <info name="Port1State" value="4"/>
              <info name="Port1LID" value="0xa"/>
              <info name="Port1LMC" value="0"/>
              <info name="Port1GID0" 
value="fe80:0000:0000:0000:0011:7500:0076:f186"/>
            </object>
          </object>
        </object>
      </object>
    </object>
  </object>
</topology>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE topology SYSTEM "hwloc.dtd">
<topology>
  <object type="Machine" os_index="0" cpuset="0x00000058" 
complete_cpuset="0x0000ffff" online_cpuset="0x0000ffff" 
allowed_cpuset="0x00000058" nodeset="0x00000003" complete_nodeset="0x00000003" 
allowed_nodeset="0x00000003">
    <page_type size="4096" count="0"/>
    <page_type size="2097152" count="0"/>
    <info name="DMIProductName" value="PowerEdge R720"/>
    <info name="DMIProductVersion" value=""/>
    <info name="DMIBoardVendor" value="Dell Inc."/>
    <info name="DMIBoardName" value="046V88"/>
    <info name="DMIBoardVersion" value="A00"/>
    <info name="DMIBoardAssetTag" value=""/>
    <info name="DMIChassisVendor" value="Dell Inc."/>
    <info name="DMIChassisType" value="23"/>
    <info name="DMIChassisVersion" value=""/>
    <info name="DMIChassisAssetTag" value=""/>
    <info name="DMIBIOSVendor" value="Dell Inc."/>
    <info name="DMIBIOSVersion" value="1.2.6"/>
    <info name="DMIBIOSDate" value="05/10/2012"/>
    <info name="DMISysVendor" value="Dell Inc."/>
    <info name="Backend" value="Linux"/>
    <info name="LinuxCgroup" value="/slurm/uid_11922/job_106635/step_0"/>
    <info name="OSName" value="Linux"/>
    <info name="OSRelease" value="3.13.0-39-generic"/>
    <info name="OSVersion" value="#66-Ubuntu SMP Tue Oct 28 13:30:27 UTC 2014"/>
    <info name="HostName" value="coma03"/>
    <info name="Architecture" value="x86_64"/>
    <distances nbobjs="2" relative_depth="1" latency_base="10.000000">
      <latency value="1.000000"/>
      <latency value="2.000000"/>
      <latency value="2.000000"/>
      <latency value="1.000000"/>
    </distances>
    <object type="NUMANode" os_index="0" cpuset="0x00000050" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000050" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" local_memory="67514843136">
      <page_type size="4096" count="16483116"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="0" cpuset="0x00000050" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000050" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000050" complete_cpuset="0x00005555" 
online_cpuset="0x00005555" allowed_cpuset="0x00000050" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000010" complete_cpuset="0x00000010" 
online_cpuset="0x00000010" allowed_cpuset="0x00000010" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="2" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="4" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
          <object type="Cache" cpuset="0x00000040" complete_cpuset="0x00000040" 
online_cpuset="0x00000040" allowed_cpuset="0x00000040" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="3" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="6" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="0" bridge_type="0-1" depth="0" 
bridge_pci="0000:[00-0d]">
        <object type="Bridge" os_index="16" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1a" bridge_type="1-1" depth="0" 
bridge_pci="0000:[02-02]" pci_busid="0000:00:01.0" pci_type="0604 [8086:3c02] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1a"/>
          <object type="PCIDev" os_index="8192" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em3" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d7"/>
            </object>
          </object>
          <object type="PCIDev" os_index="8193" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em4" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d8"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="17" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1b" bridge_type="1-1" depth="0" 
bridge_pci="0000:[01-01]" pci_busid="0000:00:01.1" pci_type="0604 [8086:3c03] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1b"/>
          <object type="PCIDev" os_index="4096" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em1" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d5"/>
            </object>
          </object>
          <object type="PCIDev" os_index="4097" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em2" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d6"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="34" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 2c" bridge_type="1-1" depth="0" 
bridge_pci="0000:[03-03]" pci_busid="0000:00:02.2" pci_type="0604 [8086:3c06] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 2c"/>
          <object type="PCIDev" os_index="12288" name="LSI Logic / Symbios 
Logic MegaRAID SAS 2208 [Thunderbolt]" pci_busid="0000:03:00.0" pci_type="0104 
[1000:005b] [1028:1f38] 01" pci_link_speed="0.000000">
            <info name="PCIVendor" value="LSI Logic / Symbios Logic"/>
            <info name="PCIDevice" value="MegaRAID SAS 2208 [Thunderbolt]"/>
            <object type="OSDev" name="sda" osdev_type="0"/>
          </object>
        </object>
        <object type="Bridge" os_index="48" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[05-05]" pci_busid="0000:00:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="20480" name="NVIDIA Corporation 
GF110GL [Tesla M2090]" pci_busid="0000:05:00.0" pci_type="0302 [10de:1091] 
[10de:0887] a1" pci_link_speed="0.000000">
            <info name="PCIVendor" value="NVIDIA Corporation"/>
            <info name="PCIDevice" value="GF110GL [Tesla M2090]"/>
            <object type="OSDev" name="card0" osdev_type="1"/>
          </object>
        </object>
        <object type="Bridge" os_index="455" name="Intel Corporation C600/X79 
series chipset PCI Express Root Port 8" bridge_type="1-1" depth="0" 
bridge_pci="0000:[08-0c]" pci_busid="0000:00:1c.7" pci_type="0604 [8086:1d1e] 
[0000:0000] b5" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="C600/X79 series chipset PCI Express 
Root Port 8"/>
          <object type="Bridge" os_index="32768" name="Renesas Technology Corp. 
SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="1" bridge_pci="0000:[09-0c]" 
pci_busid="0000:08:00.0" pci_type="0604 [1912:0013] [0000:0000] 00" 
pci_link_speed="0.000000">
            <info name="PCIVendor" value="Renesas Technology Corp."/>
            <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
            <object type="Bridge" os_index="36864" name="Renesas Technology 
Corp. SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="2" 
bridge_pci="0000:[0a-0b]" pci_busid="0000:09:00.0" pci_type="0604 [1912:0013] 
[0000:0000] 00" pci_link_speed="0.000000">
              <info name="PCIVendor" value="Renesas Technology Corp."/>
              <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
              <object type="Bridge" os_index="40960" name="Renesas Technology 
Corp. SH7757 PCIe-PCI Bridge [PPB]" bridge_type="1-1" depth="3" 
bridge_pci="0000:[0b-0b]" pci_busid="0000:0a:00.0" pci_type="0604 [1912:0012] 
[0000:0000] 00" pci_link_speed="0.000000">
                <info name="PCIVendor" value="Renesas Technology Corp."/>
                <info name="PCIDevice" value="SH7757 PCIe-PCI Bridge [PPB]"/>
                <object type="PCIDev" os_index="45056" name="Matrox Electronics 
Systems Ltd. G200eR2" pci_busid="0000:0b:00.0" pci_type="0300 [102b:0534] 
[1028:048c] 00" pci_link_speed="0.000000">
                  <info name="PCIVendor" value="Matrox Electronics Systems 
Ltd."/>
                  <info name="PCIDevice" value="G200eR2"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
    </object>
    <object type="NUMANode" os_index="1" cpuset="0x00000008" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" local_memory="67643228160">
      <page_type size="4096" count="16514460"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="1" cpuset="0x00000008" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000008" complete_cpuset="0x0000aaaa" 
online_cpuset="0x0000aaaa" allowed_cpuset="0x00000008" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000008" complete_cpuset="0x00000008" 
online_cpuset="0x00000008" allowed_cpuset="0x00000008" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="1" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
                  <object type="PU" os_index="3" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="2" bridge_type="0-1" depth="0" 
bridge_pci="0000:[40-44]">
        <object type="Bridge" os_index="262192" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[43-43]" pci_busid="0000:40:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="274432" name="QLogic Corp. IBA7322 
QDR InfiniBand HCA" pci_busid="0000:43:00.0" pci_type="0c06 [1077:7322] 
[1077:7322] 02" pci_link_speed="0.000000">
            <info name="PCIVendor" value="QLogic Corp."/>
            <info name="PCIDevice" value="IBA7322 QDR InfiniBand HCA"/>
            <object type="OSDev" name="ib0" osdev_type="2">
              <info name="Address" 
value="80:00:00:03:fe:80:00:00:00:00:00:00:00:11:75:00:00:76:f1:86"/>
              <info name="Port" value="1"/>
            </object>
            <object type="OSDev" name="qib0" osdev_type="3">
              <info name="NodeGUID" value="0011:7500:0076:f186"/>
              <info name="SysImageGUID" value="0011:7500:0076:f186"/>
              <info name="Port1State" value="4"/>
              <info name="Port1LID" value="0xa"/>
              <info name="Port1LMC" value="0"/>
              <info name="Port1GID0" 
value="fe80:0000:0000:0000:0011:7500:0076:f186"/>
            </object>
          </object>
        </object>
      </object>
    </object>
  </object>
</topology>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE topology SYSTEM "hwloc.dtd">
<topology>
  <object type="Machine" os_index="0" cpuset="0x00000058" 
complete_cpuset="0x0000ffff" online_cpuset="0x0000ffff" 
allowed_cpuset="0x00000058" nodeset="0x00000003" complete_nodeset="0x00000003" 
allowed_nodeset="0x00000003">
    <page_type size="4096" count="0"/>
    <page_type size="2097152" count="0"/>
    <info name="DMIProductName" value="PowerEdge R720"/>
    <info name="DMIProductVersion" value=""/>
    <info name="DMIBoardVendor" value="Dell Inc."/>
    <info name="DMIBoardName" value="046V88"/>
    <info name="DMIBoardVersion" value="A00"/>
    <info name="DMIBoardAssetTag" value=""/>
    <info name="DMIChassisVendor" value="Dell Inc."/>
    <info name="DMIChassisType" value="23"/>
    <info name="DMIChassisVersion" value=""/>
    <info name="DMIChassisAssetTag" value=""/>
    <info name="DMIBIOSVendor" value="Dell Inc."/>
    <info name="DMIBIOSVersion" value="1.2.6"/>
    <info name="DMIBIOSDate" value="05/10/2012"/>
    <info name="DMISysVendor" value="Dell Inc."/>
    <info name="Backend" value="Linux"/>
    <info name="LinuxCgroup" value="/slurm/uid_11922/job_106635/step_0"/>
    <info name="OSName" value="Linux"/>
    <info name="OSRelease" value="3.13.0-39-generic"/>
    <info name="OSVersion" value="#66-Ubuntu SMP Tue Oct 28 13:30:27 UTC 2014"/>
    <info name="HostName" value="coma03"/>
    <info name="Architecture" value="x86_64"/>
    <distances nbobjs="2" relative_depth="1" latency_base="10.000000">
      <latency value="1.000000"/>
      <latency value="2.000000"/>
      <latency value="2.000000"/>
      <latency value="1.000000"/>
    </distances>
    <object type="NUMANode" os_index="0" cpuset="0x00000050" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000050" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" local_memory="67514843136">
      <page_type size="4096" count="16483116"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="0" cpuset="0x00000050" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000050" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000050" complete_cpuset="0x00005555" 
online_cpuset="0x00005555" allowed_cpuset="0x00000050" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000010" complete_cpuset="0x00000010" 
online_cpuset="0x00000010" allowed_cpuset="0x00000010" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="2" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="4" cpuset="0x00000010" 
complete_cpuset="0x00000010" online_cpuset="0x00000010" 
allowed_cpuset="0x00000010" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
          <object type="Cache" cpuset="0x00000040" complete_cpuset="0x00000040" 
online_cpuset="0x00000040" allowed_cpuset="0x00000040" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="3" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="6" cpuset="0x00000040" 
complete_cpuset="0x00000040" online_cpuset="0x00000040" 
allowed_cpuset="0x00000040" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="0" bridge_type="0-1" depth="0" 
bridge_pci="0000:[00-0d]">
        <object type="Bridge" os_index="16" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1a" bridge_type="1-1" depth="0" 
bridge_pci="0000:[02-02]" pci_busid="0000:00:01.0" pci_type="0604 [8086:3c02] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1a"/>
          <object type="PCIDev" os_index="8192" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em3" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d7"/>
            </object>
          </object>
          <object type="PCIDev" os_index="8193" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em4" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d8"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="17" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1b" bridge_type="1-1" depth="0" 
bridge_pci="0000:[01-01]" pci_busid="0000:00:01.1" pci_type="0604 [8086:3c03] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1b"/>
          <object type="PCIDev" os_index="4096" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em1" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d5"/>
            </object>
          </object>
          <object type="PCIDev" os_index="4097" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em2" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d6"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="34" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 2c" bridge_type="1-1" depth="0" 
bridge_pci="0000:[03-03]" pci_busid="0000:00:02.2" pci_type="0604 [8086:3c06] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 2c"/>
          <object type="PCIDev" os_index="12288" name="LSI Logic / Symbios 
Logic MegaRAID SAS 2208 [Thunderbolt]" pci_busid="0000:03:00.0" pci_type="0104 
[1000:005b] [1028:1f38] 01" pci_link_speed="0.000000">
            <info name="PCIVendor" value="LSI Logic / Symbios Logic"/>
            <info name="PCIDevice" value="MegaRAID SAS 2208 [Thunderbolt]"/>
            <object type="OSDev" name="sda" osdev_type="0"/>
          </object>
        </object>
        <object type="Bridge" os_index="48" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[05-05]" pci_busid="0000:00:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="20480" name="NVIDIA Corporation 
GF110GL [Tesla M2090]" pci_busid="0000:05:00.0" pci_type="0302 [10de:1091] 
[10de:0887] a1" pci_link_speed="0.000000">
            <info name="PCIVendor" value="NVIDIA Corporation"/>
            <info name="PCIDevice" value="GF110GL [Tesla M2090]"/>
            <object type="OSDev" name="card0" osdev_type="1"/>
          </object>
        </object>
        <object type="Bridge" os_index="455" name="Intel Corporation C600/X79 
series chipset PCI Express Root Port 8" bridge_type="1-1" depth="0" 
bridge_pci="0000:[08-0c]" pci_busid="0000:00:1c.7" pci_type="0604 [8086:1d1e] 
[0000:0000] b5" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="C600/X79 series chipset PCI Express 
Root Port 8"/>
          <object type="Bridge" os_index="32768" name="Renesas Technology Corp. 
SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="1" bridge_pci="0000:[09-0c]" 
pci_busid="0000:08:00.0" pci_type="0604 [1912:0013] [0000:0000] 00" 
pci_link_speed="0.000000">
            <info name="PCIVendor" value="Renesas Technology Corp."/>
            <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
            <object type="Bridge" os_index="36864" name="Renesas Technology 
Corp. SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="2" 
bridge_pci="0000:[0a-0b]" pci_busid="0000:09:00.0" pci_type="0604 [1912:0013] 
[0000:0000] 00" pci_link_speed="0.000000">
              <info name="PCIVendor" value="Renesas Technology Corp."/>
              <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
              <object type="Bridge" os_index="40960" name="Renesas Technology 
Corp. SH7757 PCIe-PCI Bridge [PPB]" bridge_type="1-1" depth="3" 
bridge_pci="0000:[0b-0b]" pci_busid="0000:0a:00.0" pci_type="0604 [1912:0012] 
[0000:0000] 00" pci_link_speed="0.000000">
                <info name="PCIVendor" value="Renesas Technology Corp."/>
                <info name="PCIDevice" value="SH7757 PCIe-PCI Bridge [PPB]"/>
                <object type="PCIDev" os_index="45056" name="Matrox Electronics 
Systems Ltd. G200eR2" pci_busid="0000:0b:00.0" pci_type="0300 [102b:0534] 
[1028:048c] 00" pci_link_speed="0.000000">
                  <info name="PCIVendor" value="Matrox Electronics Systems 
Ltd."/>
                  <info name="PCIDevice" value="G200eR2"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
    </object>
    <object type="NUMANode" os_index="1" cpuset="0x00000008" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" local_memory="67643228160">
      <page_type size="4096" count="16514460"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="1" cpuset="0x00000008" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000008" complete_cpuset="0x0000aaaa" 
online_cpuset="0x0000aaaa" allowed_cpuset="0x00000008" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000008" complete_cpuset="0x00000008" 
online_cpuset="0x00000008" allowed_cpuset="0x00000008" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="1" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
                  <object type="PU" os_index="3" cpuset="0x00000008" 
complete_cpuset="0x00000008" online_cpuset="0x00000008" 
allowed_cpuset="0x00000008" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="2" bridge_type="0-1" depth="0" 
bridge_pci="0000:[40-44]">
        <object type="Bridge" os_index="262192" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[43-43]" pci_busid="0000:40:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="274432" name="QLogic Corp. IBA7322 
QDR InfiniBand HCA" pci_busid="0000:43:00.0" pci_type="0c06 [1077:7322] 
[1077:7322] 02" pci_link_speed="0.000000">
            <info name="PCIVendor" value="QLogic Corp."/>
            <info name="PCIDevice" value="IBA7322 QDR InfiniBand HCA"/>
            <object type="OSDev" name="ib0" osdev_type="2">
              <info name="Address" 
value="80:00:00:03:fe:80:00:00:00:00:00:00:00:11:75:00:00:76:f1:86"/>
              <info name="Port" value="1"/>
            </object>
            <object type="OSDev" name="qib0" osdev_type="3">
              <info name="NodeGUID" value="0011:7500:0076:f186"/>
              <info name="SysImageGUID" value="0011:7500:0076:f186"/>
              <info name="Port1State" value="4"/>
              <info name="Port1LID" value="0xa"/>
              <info name="Port1LMC" value="0"/>
              <info name="Port1GID0" 
value="fe80:0000:0000:0000:0011:7500:0076:f186"/>
            </object>
          </object>
        </object>
      </object>
    </object>
  </object>
</topology>
Begin: init_MPI
MPI_Init....
Begin: init_MPI
MPI_Init....
Begin: init_MPI
MPI_Init....
****************************************************************************
* hwloc has encountered an out-of-order topology discovery.
* An object with cpuset 0x00000008 was inserted after object with 0x00000050
* Please check that your input topology (XML file, etc.) is valid.
****************************************************************************
Begin: init_MPI
MPI_Init....
****************************************************************************
* hwloc has encountered an out-of-order topology discovery.
* An object with cpuset 0x00000008 was inserted after object with 0x00000050
* Please check that your input topology (XML file, etc.) is valid.
****************************************************************************
****************************************************************************
* hwloc has encountered an out-of-order topology discovery.
* An object with cpuset 0x00000008 was inserted after object with 0x00000050
* Please check that your input topology (XML file, etc.) is valid.
****************************************************************************
MPI_Comm_size....
MPI_Comm_size....
MPI_Comm_rank....
MPI_Get_processor_name...
myargs....
MYID = 0 
numprocs=4  ncpux[1]=2  ncpux[2]=2  ncpux[3]=1  percpusize:   NM1=32 NM2=32 
NM3=32 proc: .0000 on coma03
end: init_MPI

user code output begins here...

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE topology SYSTEM "hwloc.dtd">
<topology>
  <object type="Machine" os_index="0" cpuset="0x00000007" 
complete_cpuset="0x0000ffff" online_cpuset="0x0000ffff" 
allowed_cpuset="0x00000007" nodeset="0x00000003" complete_nodeset="0x00000003" 
allowed_nodeset="0x00000003">
    <page_type size="4096" count="0"/>
    <page_type size="2097152" count="0"/>
    <info name="DMIProductName" value="PowerEdge R720"/>
    <info name="DMIProductVersion" value=""/>
    <info name="DMIBoardVendor" value="Dell Inc."/>
    <info name="DMIBoardName" value="046V88"/>
    <info name="DMIBoardVersion" value="A00"/>
    <info name="DMIBoardAssetTag" value=""/>
    <info name="DMIChassisVendor" value="Dell Inc."/>
    <info name="DMIChassisType" value="23"/>
    <info name="DMIChassisVersion" value=""/>
    <info name="DMIChassisAssetTag" value=""/>
    <info name="DMIBIOSVendor" value="Dell Inc."/>
    <info name="DMIBIOSVersion" value="1.2.6"/>
    <info name="DMIBIOSDate" value="05/10/2012"/>
    <info name="DMISysVendor" value="Dell Inc."/>
    <info name="Backend" value="Linux"/>
    <info name="LinuxCgroup" value="/slurm/uid_11922/job_106634/step_0"/>
    <info name="OSName" value="Linux"/>
    <info name="OSRelease" value="3.13.0-39-generic"/>
    <info name="OSVersion" value="#66-Ubuntu SMP Tue Oct 28 13:30:27 UTC 2014"/>
    <info name="HostName" value="coma03"/>
    <info name="Architecture" value="x86_64"/>
    <distances nbobjs="2" relative_depth="1" latency_base="10.000000">
      <latency value="1.000000"/>
      <latency value="2.000000"/>
      <latency value="2.000000"/>
      <latency value="1.000000"/>
    </distances>
    <object type="NUMANode" os_index="0" cpuset="0x00000005" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000005" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" local_memory="67514843136">
      <page_type size="4096" count="16483116"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="0" cpuset="0x00000005" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000005" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000005" complete_cpuset="0x00005555" 
online_cpuset="0x00005555" allowed_cpuset="0x00000005" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000001" complete_cpuset="0x00000001" 
online_cpuset="0x00000001" allowed_cpuset="0x00000001" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="0" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="0" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
          <object type="Cache" cpuset="0x00000004" complete_cpuset="0x00000004" 
online_cpuset="0x00000004" allowed_cpuset="0x00000004" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="1" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="2" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="0" bridge_type="0-1" depth="0" 
bridge_pci="0000:[00-0d]">
        <object type="Bridge" os_index="16" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1a" bridge_type="1-1" depth="0" 
bridge_pci="0000:[02-02]" pci_busid="0000:00:01.0" pci_type="0604 [8086:3c02] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1a"/>
          <object type="PCIDev" os_index="8192" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em3" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d7"/>
            </object>
          </object>
          <object type="PCIDev" os_index="8193" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em4" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d8"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="17" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1b" bridge_type="1-1" depth="0" 
bridge_pci="0000:[01-01]" pci_busid="0000:00:01.1" pci_type="0604 [8086:3c03] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1b"/>
          <object type="PCIDev" os_index="4096" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em1" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d5"/>
            </object>
          </object>
          <object type="PCIDev" os_index="4097" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em2" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d6"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="34" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 2c" bridge_type="1-1" depth="0" 
bridge_pci="0000:[03-03]" pci_busid="0000:00:02.2" pci_type="0604 [8086:3c06] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 2c"/>
          <object type="PCIDev" os_index="12288" name="LSI Logic / Symbios 
Logic MegaRAID SAS 2208 [Thunderbolt]" pci_busid="0000:03:00.0" pci_type="0104 
[1000:005b] [1028:1f38] 01" pci_link_speed="0.000000">
            <info name="PCIVendor" value="LSI Logic / Symbios Logic"/>
            <info name="PCIDevice" value="MegaRAID SAS 2208 [Thunderbolt]"/>
            <object type="OSDev" name="sda" osdev_type="0"/>
          </object>
        </object>
        <object type="Bridge" os_index="48" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[05-05]" pci_busid="0000:00:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="20480" name="NVIDIA Corporation 
GF110GL [Tesla M2090]" pci_busid="0000:05:00.0" pci_type="0302 [10de:1091] 
[10de:0887] a1" pci_link_speed="0.000000">
            <info name="PCIVendor" value="NVIDIA Corporation"/>
            <info name="PCIDevice" value="GF110GL [Tesla M2090]"/>
            <object type="OSDev" name="card0" osdev_type="1"/>
          </object>
        </object>
        <object type="Bridge" os_index="455" name="Intel Corporation C600/X79 
series chipset PCI Express Root Port 8" bridge_type="1-1" depth="0" 
bridge_pci="0000:[08-0c]" pci_busid="0000:00:1c.7" pci_type="0604 [8086:1d1e] 
[0000:0000] b5" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="C600/X79 series chipset PCI Express 
Root Port 8"/>
          <object type="Bridge" os_index="32768" name="Renesas Technology Corp. 
SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="1" bridge_pci="0000:[09-0c]" 
pci_busid="0000:08:00.0" pci_type="0604 [1912:0013] [0000:0000] 00" 
pci_link_speed="0.000000">
            <info name="PCIVendor" value="Renesas Technology Corp."/>
            <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
            <object type="Bridge" os_index="36864" name="Renesas Technology 
Corp. SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="2" 
bridge_pci="0000:[0a-0b]" pci_busid="0000:09:00.0" pci_type="0604 [1912:0013] 
[0000:0000] 00" pci_link_speed="0.000000">
              <info name="PCIVendor" value="Renesas Technology Corp."/>
              <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
              <object type="Bridge" os_index="40960" name="Renesas Technology 
Corp. SH7757 PCIe-PCI Bridge [PPB]" bridge_type="1-1" depth="3" 
bridge_pci="0000:[0b-0b]" pci_busid="0000:0a:00.0" pci_type="0604 [1912:0012] 
[0000:0000] 00" pci_link_speed="0.000000">
                <info name="PCIVendor" value="Renesas Technology Corp."/>
                <info name="PCIDevice" value="SH7757 PCIe-PCI Bridge [PPB]"/>
                <object type="PCIDev" os_index="45056" name="Matrox Electronics 
Systems Ltd. G200eR2" pci_busid="0000:0b:00.0" pci_type="0300 [102b:0534] 
[1028:048c] 00" pci_link_speed="0.000000">
                  <info name="PCIVendor" value="Matrox Electronics Systems 
Ltd."/>
                  <info name="PCIDevice" value="G200eR2"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
    </object>
    <object type="NUMANode" os_index="1" cpuset="0x00000002" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" local_memory="67643228160">
      <page_type size="4096" count="16514460"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="1" cpuset="0x00000002" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000002" complete_cpuset="0x0000aaaa" 
online_cpuset="0x0000aaaa" allowed_cpuset="0x00000002" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000002" complete_cpuset="0x00000002" 
online_cpuset="0x00000002" allowed_cpuset="0x00000002" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="0" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
                  <object type="PU" os_index="1" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="2" bridge_type="0-1" depth="0" 
bridge_pci="0000:[40-44]">
        <object type="Bridge" os_index="262192" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[43-43]" pci_busid="0000:40:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="274432" name="QLogic Corp. IBA7322 
QDR InfiniBand HCA" pci_busid="0000:43:00.0" pci_type="0c06 [1077:7322] 
[1077:7322] 02" pci_link_speed="0.000000">
            <info name="PCIVendor" value="QLogic Corp."/>
            <info name="PCIDevice" value="IBA7322 QDR InfiniBand HCA"/>
            <object type="OSDev" name="ib0" osdev_type="2">
              <info name="Address" 
value="80:00:00:03:fe:80:00:00:00:00:00:00:00:11:75:00:00:76:f1:86"/>
              <info name="Port" value="1"/>
            </object>
            <object type="OSDev" name="qib0" osdev_type="3">
              <info name="NodeGUID" value="0011:7500:0076:f186"/>
              <info name="SysImageGUID" value="0011:7500:0076:f186"/>
              <info name="Port1State" value="4"/>
              <info name="Port1LID" value="0xa"/>
              <info name="Port1LMC" value="0"/>
              <info name="Port1GID0" 
value="fe80:0000:0000:0000:0011:7500:0076:f186"/>
            </object>
          </object>
        </object>
      </object>
    </object>
  </object>
</topology>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE topology SYSTEM "hwloc.dtd">
<topology>
  <object type="Machine" os_index="0" cpuset="0x00000007" 
complete_cpuset="0x0000ffff" online_cpuset="0x0000ffff" 
allowed_cpuset="0x00000007" nodeset="0x00000003" complete_nodeset="0x00000003" 
allowed_nodeset="0x00000003">
    <page_type size="4096" count="0"/>
    <page_type size="2097152" count="0"/>
    <info name="DMIProductName" value="PowerEdge R720"/>
    <info name="DMIProductVersion" value=""/>
    <info name="DMIBoardVendor" value="Dell Inc."/>
    <info name="DMIBoardName" value="046V88"/>
    <info name="DMIBoardVersion" value="A00"/>
    <info name="DMIBoardAssetTag" value=""/>
    <info name="DMIChassisVendor" value="Dell Inc."/>
    <info name="DMIChassisType" value="23"/>
    <info name="DMIChassisVersion" value=""/>
    <info name="DMIChassisAssetTag" value=""/>
    <info name="DMIBIOSVendor" value="Dell Inc."/>
    <info name="DMIBIOSVersion" value="1.2.6"/>
    <info name="DMIBIOSDate" value="05/10/2012"/>
    <info name="DMISysVendor" value="Dell Inc."/>
    <info name="Backend" value="Linux"/>
    <info name="LinuxCgroup" value="/slurm/uid_11922/job_106634/step_0"/>
    <info name="OSName" value="Linux"/>
    <info name="OSRelease" value="3.13.0-39-generic"/>
    <info name="OSVersion" value="#66-Ubuntu SMP Tue Oct 28 13:30:27 UTC 2014"/>
    <info name="HostName" value="coma03"/>
    <info name="Architecture" value="x86_64"/>
    <distances nbobjs="2" relative_depth="1" latency_base="10.000000">
      <latency value="1.000000"/>
      <latency value="2.000000"/>
      <latency value="2.000000"/>
      <latency value="1.000000"/>
    </distances>
    <object type="NUMANode" os_index="0" cpuset="0x00000005" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000005" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" local_memory="67514843136">
      <page_type size="4096" count="16483116"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="0" cpuset="0x00000005" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000005" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000005" complete_cpuset="0x00005555" 
online_cpuset="0x00005555" allowed_cpuset="0x00000005" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000001" complete_cpuset="0x00000001" 
online_cpuset="0x00000001" allowed_cpuset="0x00000001" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="0" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="0" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
          <object type="Cache" cpuset="0x00000004" complete_cpuset="0x00000004" 
online_cpuset="0x00000004" allowed_cpuset="0x00000004" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="1" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="2" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="0" bridge_type="0-1" depth="0" 
bridge_pci="0000:[00-0d]">
        <object type="Bridge" os_index="16" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1a" bridge_type="1-1" depth="0" 
bridge_pci="0000:[02-02]" pci_busid="0000:00:01.0" pci_type="0604 [8086:3c02] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1a"/>
          <object type="PCIDev" os_index="8192" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em3" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d7"/>
            </object>
          </object>
          <object type="PCIDev" os_index="8193" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em4" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d8"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="17" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1b" bridge_type="1-1" depth="0" 
bridge_pci="0000:[01-01]" pci_busid="0000:00:01.1" pci_type="0604 [8086:3c03] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1b"/>
          <object type="PCIDev" os_index="4096" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em1" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d5"/>
            </object>
          </object>
          <object type="PCIDev" os_index="4097" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em2" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d6"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="34" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 2c" bridge_type="1-1" depth="0" 
bridge_pci="0000:[03-03]" pci_busid="0000:00:02.2" pci_type="0604 [8086:3c06] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 2c"/>
          <object type="PCIDev" os_index="12288" name="LSI Logic / Symbios 
Logic MegaRAID SAS 2208 [Thunderbolt]" pci_busid="0000:03:00.0" pci_type="0104 
[1000:005b] [1028:1f38] 01" pci_link_speed="0.000000">
            <info name="PCIVendor" value="LSI Logic / Symbios Logic"/>
            <info name="PCIDevice" value="MegaRAID SAS 2208 [Thunderbolt]"/>
            <object type="OSDev" name="sda" osdev_type="0"/>
          </object>
        </object>
        <object type="Bridge" os_index="48" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[05-05]" pci_busid="0000:00:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="20480" name="NVIDIA Corporation 
GF110GL [Tesla M2090]" pci_busid="0000:05:00.0" pci_type="0302 [10de:1091] 
[10de:0887] a1" pci_link_speed="0.000000">
            <info name="PCIVendor" value="NVIDIA Corporation"/>
            <info name="PCIDevice" value="GF110GL [Tesla M2090]"/>
            <object type="OSDev" name="card0" osdev_type="1"/>
          </object>
        </object>
        <object type="Bridge" os_index="455" name="Intel Corporation C600/X79 
series chipset PCI Express Root Port 8" bridge_type="1-1" depth="0" 
bridge_pci="0000:[08-0c]" pci_busid="0000:00:1c.7" pci_type="0604 [8086:1d1e] 
[0000:0000] b5" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="C600/X79 series chipset PCI Express 
Root Port 8"/>
          <object type="Bridge" os_index="32768" name="Renesas Technology Corp. 
SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="1" bridge_pci="0000:[09-0c]" 
pci_busid="0000:08:00.0" pci_type="0604 [1912:0013] [0000:0000] 00" 
pci_link_speed="0.000000">
            <info name="PCIVendor" value="Renesas Technology Corp."/>
            <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
            <object type="Bridge" os_index="36864" name="Renesas Technology 
Corp. SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="2" 
bridge_pci="0000:[0a-0b]" pci_busid="0000:09:00.0" pci_type="0604 [1912:0013] 
[0000:0000] 00" pci_link_speed="0.000000">
              <info name="PCIVendor" value="Renesas Technology Corp."/>
              <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
              <object type="Bridge" os_index="40960" name="Renesas Technology 
Corp. SH7757 PCIe-PCI Bridge [PPB]" bridge_type="1-1" depth="3" 
bridge_pci="0000:[0b-0b]" pci_busid="0000:0a:00.0" pci_type="0604 [1912:0012] 
[0000:0000] 00" pci_link_speed="0.000000">
                <info name="PCIVendor" value="Renesas Technology Corp."/>
                <info name="PCIDevice" value="SH7757 PCIe-PCI Bridge [PPB]"/>
                <object type="PCIDev" os_index="45056" name="Matrox Electronics 
Systems Ltd. G200eR2" pci_busid="0000:0b:00.0" pci_type="0300 [102b:0534] 
[1028:048c] 00" pci_link_speed="0.000000">
                  <info name="PCIVendor" value="Matrox Electronics Systems 
Ltd."/>
                  <info name="PCIDevice" value="G200eR2"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
    </object>
    <object type="NUMANode" os_index="1" cpuset="0x00000002" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" local_memory="67643228160">
      <page_type size="4096" count="16514460"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="1" cpuset="0x00000002" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000002" complete_cpuset="0x0000aaaa" 
online_cpuset="0x0000aaaa" allowed_cpuset="0x00000002" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000002" complete_cpuset="0x00000002" 
online_cpuset="0x00000002" allowed_cpuset="0x00000002" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="0" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
                  <object type="PU" os_index="1" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="2" bridge_type="0-1" depth="0" 
bridge_pci="0000:[40-44]">
        <object type="Bridge" os_index="262192" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[43-43]" pci_busid="0000:40:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="274432" name="QLogic Corp. IBA7322 
QDR InfiniBand HCA" pci_busid="0000:43:00.0" pci_type="0c06 [1077:7322] 
[1077:7322] 02" pci_link_speed="0.000000">
            <info name="PCIVendor" value="QLogic Corp."/>
            <info name="PCIDevice" value="IBA7322 QDR InfiniBand HCA"/>
            <object type="OSDev" name="ib0" osdev_type="2">
              <info name="Address" 
value="80:00:00:03:fe:80:00:00:00:00:00:00:00:11:75:00:00:76:f1:86"/>
              <info name="Port" value="1"/>
            </object>
            <object type="OSDev" name="qib0" osdev_type="3">
              <info name="NodeGUID" value="0011:7500:0076:f186"/>
              <info name="SysImageGUID" value="0011:7500:0076:f186"/>
              <info name="Port1State" value="4"/>
              <info name="Port1LID" value="0xa"/>
              <info name="Port1LMC" value="0"/>
              <info name="Port1GID0" 
value="fe80:0000:0000:0000:0011:7500:0076:f186"/>
            </object>
          </object>
        </object>
      </object>
    </object>
  </object>
</topology>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE topology SYSTEM "hwloc.dtd">
<topology>
  <object type="Machine" os_index="0" cpuset="0x00000007" 
complete_cpuset="0x0000ffff" online_cpuset="0x0000ffff" 
allowed_cpuset="0x00000007" nodeset="0x00000003" complete_nodeset="0x00000003" 
allowed_nodeset="0x00000003">
    <page_type size="4096" count="0"/>
    <page_type size="2097152" count="0"/>
    <info name="DMIProductName" value="PowerEdge R720"/>
    <info name="DMIProductVersion" value=""/>
    <info name="DMIBoardVendor" value="Dell Inc."/>
    <info name="DMIBoardName" value="046V88"/>
    <info name="DMIBoardVersion" value="A00"/>
    <info name="DMIBoardAssetTag" value=""/>
    <info name="DMIChassisVendor" value="Dell Inc."/>
    <info name="DMIChassisType" value="23"/>
    <info name="DMIChassisVersion" value=""/>
    <info name="DMIChassisAssetTag" value=""/>
    <info name="DMIBIOSVendor" value="Dell Inc."/>
    <info name="DMIBIOSVersion" value="1.2.6"/>
    <info name="DMIBIOSDate" value="05/10/2012"/>
    <info name="DMISysVendor" value="Dell Inc."/>
    <info name="Backend" value="Linux"/>
    <info name="LinuxCgroup" value="/slurm/uid_11922/job_106634/step_0"/>
    <info name="OSName" value="Linux"/>
    <info name="OSRelease" value="3.13.0-39-generic"/>
    <info name="OSVersion" value="#66-Ubuntu SMP Tue Oct 28 13:30:27 UTC 2014"/>
    <info name="HostName" value="coma03"/>
    <info name="Architecture" value="x86_64"/>
    <distances nbobjs="2" relative_depth="1" latency_base="10.000000">
      <latency value="1.000000"/>
      <latency value="2.000000"/>
      <latency value="2.000000"/>
      <latency value="1.000000"/>
    </distances>
    <object type="NUMANode" os_index="0" cpuset="0x00000005" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000005" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" local_memory="67514843136">
      <page_type size="4096" count="16483116"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="0" cpuset="0x00000005" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000005" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000005" complete_cpuset="0x00005555" 
online_cpuset="0x00005555" allowed_cpuset="0x00000005" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000001" complete_cpuset="0x00000001" 
online_cpuset="0x00000001" allowed_cpuset="0x00000001" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="0" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="0" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
          <object type="Cache" cpuset="0x00000004" complete_cpuset="0x00000004" 
online_cpuset="0x00000004" allowed_cpuset="0x00000004" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="1" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="2" cpuset="0x00000004" 
complete_cpuset="0x00000004" online_cpuset="0x00000004" 
allowed_cpuset="0x00000004" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="0" bridge_type="0-1" depth="0" 
bridge_pci="0000:[00-0d]">
        <object type="Bridge" os_index="16" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1a" bridge_type="1-1" depth="0" 
bridge_pci="0000:[02-02]" pci_busid="0000:00:01.0" pci_type="0604 [8086:3c02] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1a"/>
          <object type="PCIDev" os_index="8192" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em3" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d7"/>
            </object>
          </object>
          <object type="PCIDev" os_index="8193" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em4" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d8"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="17" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 1b" bridge_type="1-1" depth="0" 
bridge_pci="0000:[01-01]" pci_busid="0000:00:01.1" pci_type="0604 [8086:3c03] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 1b"/>
          <object type="PCIDev" os_index="4096" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em1" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d5"/>
            </object>
          </object>
          <object type="PCIDev" os_index="4097" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
            <info name="PCIVendor" value="Broadcom Corporation"/>
            <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
            <object type="OSDev" name="em2" osdev_type="2">
              <info name="Address" value="d4:ae:52:ad:6f:d6"/>
            </object>
          </object>
        </object>
        <object type="Bridge" os_index="34" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 2c" bridge_type="1-1" depth="0" 
bridge_pci="0000:[03-03]" pci_busid="0000:00:02.2" pci_type="0604 [8086:3c06] 
[0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 2c"/>
          <object type="PCIDev" os_index="12288" name="LSI Logic / Symbios 
Logic MegaRAID SAS 2208 [Thunderbolt]" pci_busid="0000:03:00.0" pci_type="0104 
[1000:005b] [1028:1f38] 01" pci_link_speed="0.000000">
            <info name="PCIVendor" value="LSI Logic / Symbios Logic"/>
            <info name="PCIDevice" value="MegaRAID SAS 2208 [Thunderbolt]"/>
            <object type="OSDev" name="sda" osdev_type="0"/>
          </object>
        </object>
        <object type="Bridge" os_index="48" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[05-05]" pci_busid="0000:00:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="20480" name="NVIDIA Corporation 
GF110GL [Tesla M2090]" pci_busid="0000:05:00.0" pci_type="0302 [10de:1091] 
[10de:0887] a1" pci_link_speed="0.000000">
            <info name="PCIVendor" value="NVIDIA Corporation"/>
            <info name="PCIDevice" value="GF110GL [Tesla M2090]"/>
            <object type="OSDev" name="card0" osdev_type="1"/>
          </object>
        </object>
        <object type="Bridge" os_index="455" name="Intel Corporation C600/X79 
series chipset PCI Express Root Port 8" bridge_type="1-1" depth="0" 
bridge_pci="0000:[08-0c]" pci_busid="0000:00:1c.7" pci_type="0604 [8086:1d1e] 
[0000:0000] b5" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="C600/X79 series chipset PCI Express 
Root Port 8"/>
          <object type="Bridge" os_index="32768" name="Renesas Technology Corp. 
SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="1" bridge_pci="0000:[09-0c]" 
pci_busid="0000:08:00.0" pci_type="0604 [1912:0013] [0000:0000] 00" 
pci_link_speed="0.000000">
            <info name="PCIVendor" value="Renesas Technology Corp."/>
            <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
            <object type="Bridge" os_index="36864" name="Renesas Technology 
Corp. SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="2" 
bridge_pci="0000:[0a-0b]" pci_busid="0000:09:00.0" pci_type="0604 [1912:0013] 
[0000:0000] 00" pci_link_speed="0.000000">
              <info name="PCIVendor" value="Renesas Technology Corp."/>
              <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
              <object type="Bridge" os_index="40960" name="Renesas Technology 
Corp. SH7757 PCIe-PCI Bridge [PPB]" bridge_type="1-1" depth="3" 
bridge_pci="0000:[0b-0b]" pci_busid="0000:0a:00.0" pci_type="0604 [1912:0012] 
[0000:0000] 00" pci_link_speed="0.000000">
                <info name="PCIVendor" value="Renesas Technology Corp."/>
                <info name="PCIDevice" value="SH7757 PCIe-PCI Bridge [PPB]"/>
                <object type="PCIDev" os_index="45056" name="Matrox Electronics 
Systems Ltd. G200eR2" pci_busid="0000:0b:00.0" pci_type="0300 [102b:0534] 
[1028:048c] 00" pci_link_speed="0.000000">
                  <info name="PCIVendor" value="Matrox Electronics Systems 
Ltd."/>
                  <info name="PCIDevice" value="G200eR2"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
    </object>
    <object type="NUMANode" os_index="1" cpuset="0x00000002" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" local_memory="67643228160">
      <page_type size="4096" count="16514460"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="1" cpuset="0x00000002" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000002" complete_cpuset="0x0000aaaa" 
online_cpuset="0x0000aaaa" allowed_cpuset="0x00000002" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000002" complete_cpuset="0x00000002" 
online_cpuset="0x00000002" allowed_cpuset="0x00000002" nodeset="0x00000002" 
complete_nodeset="0x00000002" allowed_nodeset="0x00000002" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="0" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002">
                  <object type="PU" os_index="1" cpuset="0x00000002" 
complete_cpuset="0x00000002" online_cpuset="0x00000002" 
allowed_cpuset="0x00000002" nodeset="0x00000002" complete_nodeset="0x00000002" 
allowed_nodeset="0x00000002"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="2" bridge_type="0-1" depth="0" 
bridge_pci="0000:[40-44]">
        <object type="Bridge" os_index="262192" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="0" bridge_pci="0000:[43-43]" pci_busid="0000:40:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Intel Corporation"/>
          <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root 
Port 3a in PCI Express Mode"/>
          <object type="PCIDev" os_index="274432" name="QLogic Corp. IBA7322 
QDR InfiniBand HCA" pci_busid="0000:43:00.0" pci_type="0c06 [1077:7322] 
[1077:7322] 02" pci_link_speed="0.000000">
            <info name="PCIVendor" value="QLogic Corp."/>
            <info name="PCIDevice" value="IBA7322 QDR InfiniBand HCA"/>
            <object type="OSDev" name="ib0" osdev_type="2">
              <info name="Address" 
value="80:00:00:03:fe:80:00:00:00:00:00:00:00:11:75:00:00:76:f1:86"/>
              <info name="Port" value="1"/>
            </object>
            <object type="OSDev" name="qib0" osdev_type="3">
              <info name="NodeGUID" value="0011:7500:0076:f186"/>
              <info name="SysImageGUID" value="0011:7500:0076:f186"/>
              <info name="Port1State" value="4"/>
              <info name="Port1LID" value="0xa"/>
              <info name="Port1LMC" value="0"/>
              <info name="Port1GID0" 
value="fe80:0000:0000:0000:0011:7500:0076:f186"/>
            </object>
          </object>
        </object>
      </object>
    </object>
  </object>
</topology>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE topology SYSTEM "hwloc.dtd">
<topology>
  <object type="Machine" os_index="0" cpuset="0x00000001" 
complete_cpuset="0x0000ffff" online_cpuset="0x0000ffff" 
allowed_cpuset="0x00000001" nodeset="0x00000003" complete_nodeset="0x00000003" 
allowed_nodeset="0x00000003">
    <page_type size="4096" count="0"/>
    <page_type size="2097152" count="0"/>
    <info name="DMIProductName" value="PowerEdge R720"/>
    <info name="DMIProductVersion" value=""/>
    <info name="DMIBoardVendor" value="Dell Inc."/>
    <info name="DMIBoardName" value="046V88"/>
    <info name="DMIBoardVersion" value="A00"/>
    <info name="DMIBoardAssetTag" value=""/>
    <info name="DMIChassisVendor" value="Dell Inc."/>
    <info name="DMIChassisType" value="23"/>
    <info name="DMIChassisVersion" value=""/>
    <info name="DMIChassisAssetTag" value=""/>
    <info name="DMIBIOSVendor" value="Dell Inc."/>
    <info name="DMIBIOSVersion" value="1.2.6"/>
    <info name="DMIBIOSDate" value="05/10/2012"/>
    <info name="DMISysVendor" value="Dell Inc."/>
    <info name="Backend" value="Linux"/>
    <info name="LinuxCgroup" value="/slurm/uid_11922/job_106634/step_0"/>
    <info name="OSName" value="Linux"/>
    <info name="OSRelease" value="3.13.0-40-generic"/>
    <info name="OSVersion" value="#69-Ubuntu SMP Thu Nov 13 17:53:56 UTC 2014"/>
    <info name="HostName" value="coma04"/>
    <info name="Architecture" value="x86_64"/>
    <distances nbobjs="2" relative_depth="1" latency_base="10.000000">
      <latency value="1.000000"/>
      <latency value="2.000000"/>
      <latency value="2.000000"/>
      <latency value="1.000000"/>
    </distances>
    <object type="NUMANode" os_index="0" cpuset="0x00000001" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" local_memory="67514843136">
      <page_type size="4096" count="16483116"/>
      <page_type size="2097152" count="0"/>
      <object type="Socket" os_index="0" cpuset="0x00000001" 
complete_cpuset="0x00005555" online_cpuset="0x00005555" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
        <info name="CPUModel" value="Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz"/>
        <object type="Cache" cpuset="0x00000001" complete_cpuset="0x00005555" 
online_cpuset="0x00005555" allowed_cpuset="0x00000001" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" 
cache_size="20971520" depth="3" cache_linesize="64" cache_associativity="20" 
cache_type="0">
          <object type="Cache" cpuset="0x00000001" complete_cpuset="0x00000001" 
online_cpuset="0x00000001" allowed_cpuset="0x00000001" nodeset="0x00000001" 
complete_nodeset="0x00000001" allowed_nodeset="0x00000001" cache_size="262144" 
depth="2" cache_linesize="64" cache_associativity="8" cache_type="0">
            <object type="Cache" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="1">
              <object type="Cache" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001" cache_size="32768" depth="1" cache_linesize="64" 
cache_associativity="8" cache_type="2">
                <object type="Core" os_index="0" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001">
                  <object type="PU" os_index="0" cpuset="0x00000001" 
complete_cpuset="0x00000001" online_cpuset="0x00000001" 
allowed_cpuset="0x00000001" nodeset="0x00000001" complete_nodeset="0x00000001" 
allowed_nodeset="0x00000001"/>
                </object>
              </object>
            </object>
          </object>
        </object>
      </object>
    </object>
    <object type="NUMANode" os_index="1" cpuset="0x0" 
complete_cpuset="0x0000aaaa" online_cpuset="0x0000aaaa" allowed_cpuset="0x0" 
nodeset="0x00000002" complete_nodeset="0x00000002" allowed_nodeset="0x00000002" 
local_memory="67643228160">
      <page_type size="4096" count="16514460"/>
      <page_type size="2097152" count="0"/>
    </object>
    <object type="Bridge" os_index="0" bridge_type="0-1" depth="0" 
bridge_pci="0000:[00-0d]">
      <object type="Bridge" os_index="16" name="Intel Corporation Xeon E5/Core 
i7 IIO PCI Express Root Port 1a" bridge_type="1-1" depth="1" 
bridge_pci="0000:[02-02]" pci_busid="0000:00:01.0" pci_type="0604 [8086:3c02] 
[0000:0000] 07" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root Port 
1a"/>
        <object type="PCIDev" os_index="8192" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Broadcom Corporation"/>
          <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
          <object type="OSDev" name="em3" osdev_type="2">
            <info name="Address" value="d4:ae:52:ad:88:12"/>
          </object>
        </object>
        <object type="PCIDev" os_index="8193" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:02:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Broadcom Corporation"/>
          <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
          <object type="OSDev" name="em4" osdev_type="2">
            <info name="Address" value="d4:ae:52:ad:88:13"/>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="17" name="Intel Corporation Xeon E5/Core 
i7 IIO PCI Express Root Port 1b" bridge_type="1-1" depth="1" 
bridge_pci="0000:[01-01]" pci_busid="0000:00:01.1" pci_type="0604 [8086:3c03] 
[0000:0000] 07" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root Port 
1b"/>
        <object type="PCIDev" os_index="4096" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.0" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Broadcom Corporation"/>
          <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
          <object type="OSDev" name="em1" osdev_type="2">
            <info name="Address" value="d4:ae:52:ad:88:10"/>
          </object>
        </object>
        <object type="PCIDev" os_index="4097" name="Broadcom Corporation 
NetXtreme BCM5720 Gigabit Ethernet PCIe" pci_busid="0000:01:00.1" 
pci_type="0200 [14e4:165f] [1028:1f5b] 00" pci_link_speed="0.000000">
          <info name="PCIVendor" value="Broadcom Corporation"/>
          <info name="PCIDevice" value="NetXtreme BCM5720 Gigabit Ethernet 
PCIe"/>
          <object type="OSDev" name="em2" osdev_type="2">
            <info name="Address" value="d4:ae:52:ad:88:11"/>
          </object>
        </object>
      </object>
      <object type="Bridge" os_index="34" name="Intel Corporation Xeon E5/Core 
i7 IIO PCI Express Root Port 2c" bridge_type="1-1" depth="1" 
bridge_pci="0000:[03-03]" pci_busid="0000:00:02.2" pci_type="0604 [8086:3c06] 
[0000:0000] 07" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root Port 
2c"/>
        <object type="PCIDev" os_index="12288" name="LSI Logic / Symbios Logic 
MegaRAID SAS 2208 [Thunderbolt]" pci_busid="0000:03:00.0" pci_type="0104 
[1000:005b] [1028:1f38] 01" pci_link_speed="0.000000">
          <info name="PCIVendor" value="LSI Logic / Symbios Logic"/>
          <info name="PCIDevice" value="MegaRAID SAS 2208 [Thunderbolt]"/>
          <object type="OSDev" name="sda" osdev_type="0"/>
        </object>
      </object>
      <object type="Bridge" os_index="48" name="Intel Corporation Xeon E5/Core 
i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="1" bridge_pci="0000:[05-05]" pci_busid="0000:00:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root Port 
3a in PCI Express Mode"/>
        <object type="PCIDev" os_index="20480" name="NVIDIA Corporation GF110GL 
[Tesla M2090]" pci_busid="0000:05:00.0" pci_type="0302 [10de:1091] [10de:0887] 
a1" pci_link_speed="0.000000">
          <info name="PCIVendor" value="NVIDIA Corporation"/>
          <info name="PCIDevice" value="GF110GL [Tesla M2090]"/>
          <object type="OSDev" name="card0" osdev_type="1"/>
        </object>
      </object>
      <object type="Bridge" os_index="455" name="Intel Corporation C600/X79 
series chipset PCI Express Root Port 8" bridge_type="1-1" depth="1" 
bridge_pci="0000:[08-0c]" pci_busid="0000:00:1c.7" pci_type="0604 [8086:1d1e] 
[0000:0000] b5" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="C600/X79 series chipset PCI Express Root 
Port 8"/>
        <object type="Bridge" os_index="32768" name="Renesas Technology Corp. 
SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="2" bridge_pci="0000:[09-0c]" 
pci_busid="0000:08:00.0" pci_type="0604 [1912:0013] [0000:0000] 00" 
pci_link_speed="0.000000">
          <info name="PCIVendor" value="Renesas Technology Corp."/>
          <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
          <object type="Bridge" os_index="36864" name="Renesas Technology Corp. 
SH7757 PCIe Switch [PS]" bridge_type="1-1" depth="3" bridge_pci="0000:[0a-0b]" 
pci_busid="0000:09:00.0" pci_type="0604 [1912:0013] [0000:0000] 00" 
pci_link_speed="0.000000">
            <info name="PCIVendor" value="Renesas Technology Corp."/>
            <info name="PCIDevice" value="SH7757 PCIe Switch [PS]"/>
            <object type="Bridge" os_index="40960" name="Renesas Technology 
Corp. SH7757 PCIe-PCI Bridge [PPB]" bridge_type="1-1" depth="4" 
bridge_pci="0000:[0b-0b]" pci_busid="0000:0a:00.0" pci_type="0604 [1912:0012] 
[0000:0000] 00" pci_link_speed="0.000000">
              <info name="PCIVendor" value="Renesas Technology Corp."/>
              <info name="PCIDevice" value="SH7757 PCIe-PCI Bridge [PPB]"/>
              <object type="PCIDev" os_index="45056" name="Matrox Electronics 
Systems Ltd. G200eR2" pci_busid="0000:0b:00.0" pci_type="0300 [102b:0534] 
[1028:048c] 00" pci_link_speed="0.000000">
                <info name="PCIVendor" value="Matrox Electronics Systems Ltd."/>
                <info name="PCIDevice" value="G200eR2"/>
              </object>
            </object>
          </object>
        </object>
      </object>
    </object>
    <object type="Bridge" os_index="2" bridge_type="0-1" depth="0" 
bridge_pci="0000:[40-44]">
      <object type="Bridge" os_index="262192" name="Intel Corporation Xeon 
E5/Core i7 IIO PCI Express Root Port 3a in PCI Express Mode" bridge_type="1-1" 
depth="1" bridge_pci="0000:[43-43]" pci_busid="0000:40:03.0" pci_type="0604 
[8086:3c08] [0000:0000] 07" pci_link_speed="0.000000">
        <info name="PCIVendor" value="Intel Corporation"/>
        <info name="PCIDevice" value="Xeon E5/Core i7 IIO PCI Express Root Port 
3a in PCI Express Mode"/>
        <object type="PCIDev" os_index="274432" name="QLogic Corp. IBA7322 QDR 
InfiniBand HCA" pci_busid="0000:43:00.0" pci_type="0c06 [1077:7322] [1077:7322] 
02" pci_link_speed="0.000000">
          <info name="PCIVendor" value="QLogic Corp."/>
          <info name="PCIDevice" value="IBA7322 QDR InfiniBand HCA"/>
          <object type="OSDev" name="ib0" osdev_type="2">
            <info name="Address" 
value="80:00:00:03:fe:80:00:00:00:00:00:00:00:11:75:00:00:76:f4:80"/>
            <info name="Port" value="1"/>
          </object>
          <object type="OSDev" name="qib0" osdev_type="3">
            <info name="NodeGUID" value="0011:7500:0076:f480"/>
            <info name="SysImageGUID" value="0011:7500:0076:f480"/>
            <info name="Port1State" value="4"/>
            <info name="Port1LID" value="0x8"/>
            <info name="Port1LMC" value="0"/>
            <info name="Port1GID0" 
value="fe80:0000:0000:0000:0011:7500:0076:f480"/>
          </object>
        </object>
      </object>
    </object>
  </object>
</topology>
Begin: init_MPI
MPI_Init....
Begin: init_MPI
MPI_Init....
Begin: init_MPI
MPI_Init....
Begin: init_MPI
MPI_Init....
MPI_Comm_size....
MPI_Comm_size....
MPI_Comm_rank....
MPI_Get_processor_name...
myargs....
MYID = 1 
numprocs=4  ncpux[1]=2  ncpux[2]=2  ncpux[3]=1  percpusize:   NM1=32 NM2=32 
NM3=32 proc: .0001 on coma03
end: init_MPI
MPI_Comm_rank....
MPI_Get_processor_name...
myargs....
MYID = 0 
numprocs=4  ncpux[1]=2  ncpux[2]=2  ncpux[3]=1  percpusize:   NM1=32 NM2=32 
NM3=32 proc: .0000 on coma03
end: init_MPI

################# MPI DOMAIN INFO ################################
MPI-DOM: pid=1 : ncpux[0-3]     =    1    2    2    1 
MPI-DOM: pid=1 : cpupos[0-3]    =    0    1    0    0 
MPI-DOM: pid=1 : totalsize[0-3] =    1   64   64   32 
MPI-DOM: pid=1 : globalpos[0-3] =    0   32    0    0 
MPI-DOM: pid=1 : n_cells_glob   = 131072 
MPI-DOM: pid=1 : global_index   = 65536 
MPI-DOM: pid=1 : numprocs       = 4 
MPI-DOM: pid=1 : n_spatial_dims = 3 
###################################################################

################# MPI DOMAIN INFO ################################
MPI_Comm_size....
MPI_Comm_rank....
MPI_Get_processor_name...
MPI-DOM: pid=0 : ncpux[0-3]     =    1    2    2    1 
myargs....
MYID = 3 
numprocs=4  ncpux[1]=2  ncpux[2]=2  ncpux[3]=1  percpusize:   NM1=32 NM2=32 
NM3=32 proc: .0003 on coma04
end: init_MPI
MPI-DOM: pid=0 : cpupos[0-3]    =    0    0    0    0 
MPI-DOM: pid=0 : totalsize[0-3] =    1   64   64   32 
MPI-DOM: pid=0 : globalpos[0-3] =    0    0    0    0 
MPI-DOM: pid=0 : n_cells_glob   = 131072 
MPI-DOM: pid=0 : global_index   = 0 
MPI-DOM: pid=0 : numprocs       = 4 
MPI-DOM: pid=0 : n_spatial_dims = 3 
###################################################################
MPI_Comm_size....
MPI_Comm_rank....
MPI_Get_processor_name...
myargs....
MYID = 2 
numprocs=4  ncpux[1]=2  ncpux[2]=2  ncpux[3]=1  percpusize:   NM1=32 NM2=32 
NM3=32 proc: .0002 on coma03
end: init_MPI

user code output begins here...


> On 09 Dec 2014, at 09:38, Gilles Gouaillardet <gilles.gouaillar...@iferc.org> 
> wrote:
> 
> Pim,
> 
> if you configure OpenMPI with --with-hwloc=external (or something like
> --with-hwloc=/usr) it is very likely
> OpenMPI will use the same hwloc library (e.g. the "system" library) that
> is used by SLURM
> 
> /* i do not know how Ubuntu packages OpenMPI ... */
> 
> 
> The default (e.g. no --with-hwloc parameter in the configure command
> line) is to use the hwloc library that is embedded within OpenMPI
> 
> Gilles
> 
> On 2014/12/09 17:34, Pim Schellart wrote:
>> Ah, ok so that was where the confusion came from, I did see hwloc in the 
>> SLURM sources but couldn’t immediately figure out where exactly it was used. 
>> We will try compiling openmpi with the embedded hwloc. Any particular flags 
>> I should set?
>> 
>>> On 09 Dec 2014, at 09:30, Ralph Castain <r...@open-mpi.org> wrote:
>>> 
>>> There is no linkage between slurm and ompi when it comes to hwloc. If you 
>>> directly launch your app using srun, then slurm will use its version of 
>>> hwloc to do the binding. If you use mpirun to launch the app, then we’ll 
>>> use our internal version to do it.
>>> 
>>> The two are completely isolated from each other.
>>> 
>>> 
>>>> On Dec 9, 2014, at 12:25 AM, Pim Schellart <p.schell...@gmail.com> wrote:
>>>> 
>>>> The version that “lstopo --version” reports is the same (1.8) on all 
>>>> nodes, but we may indeed be hitting the second issue. We can try to 
>>>> compile a new version of openmpi, but how do we ensure that the external 
>>>> programs (e.g. SLURM) are using the same hwloc version as the one embedded 
>>>> in openmpi? Is it enough to just compile hwloc 1.9 separately as well and 
>>>> link against that? Also, if this is an issue, should we file a bug against 
>>>> hwloc or openmpi on Ubuntu for mismatching versions?
>>>> 
>>>>> On 09 Dec 2014, at 00:50, Ralph Castain <r...@open-mpi.org> wrote:
>>>>> 
>>>>> Hmmm…they probably linked that to the external, system hwloc version, so 
>>>>> it sounds like one or more of your nodes has a different hwloc rpm on it.
>>>>> 
>>>>> I couldn’t leaf thru your output well enough to see all the lstopo 
>>>>> versions, but you might check to ensure they are the same.
>>>>> 
>>>>> Looking at the code base, you may also hit a problem here. OMPI 1.6 
>>>>> series was based on hwloc 1.3 - the output you sent indicated you have 
>>>>> hwloc 1.8, which is quite a big change. OMPI 1.8 series is based on hwloc 
>>>>> 1.9, so at least that is closer (though probably still a mismatch).
>>>>> 
>>>>> Frankly, I’d just download and install an OMPI tarball myself and avoid 
>>>>> these headaches. This mismatch in required versions is why we embed hwloc 
>>>>> as it is a critical library for OMPI, and we had to ensure that the 
>>>>> version matched our internal requirements.
>>>>> 
>>>>> 
>>>>>> On Dec 8, 2014, at 8:50 AM, Pim Schellart <p.schell...@gmail.com> wrote:
>>>>>> 
>>>>>> It is the default openmpi that comes with Ubuntu 14.04.
>>>>>> 
>>>>>>> On 08 Dec 2014, at 17:17, Ralph Castain <r...@open-mpi.org> wrote:
>>>>>>> 
>>>>>>> Pim: is this an OMPI you built, or one you were given somehow? If you 
>>>>>>> built it, how did you configure it?
>>>>>>> 
>>>>>>>> On Dec 8, 2014, at 8:12 AM, Brice Goglin <brice.gog...@inria.fr> wrote:
>>>>>>>> 
>>>>>>>> It likely depends on how SLURM allocates the cpuset/cgroup inside the
>>>>>>>> nodes. The XML warning is related to these restrictions inside the 
>>>>>>>> node.
>>>>>>>> Anyway, my feeling is that there's a old OMPI or a old hwloc somewhere.
>>>>>>>> 
>>>>>>>> How do we check after install whether OMPI uses the embedded or the
>>>>>>>> system-wide hwloc?
>>>>>>>> 
>>>>>>>> Brice
>>>>>>>> 
>>>>>>>> 
>>>>>>>> 
>>>>>>>> 
>>>>>>>> Le 08/12/2014 17:07, Pim Schellart a écrit :
>>>>>>>>> Dear Ralph,
>>>>>>>>> 
>>>>>>>>> the nodes are called coma## and as you can see in the logs the nodes 
>>>>>>>>> of the broken example are the same as the nodes of the working one, 
>>>>>>>>> so that doesn’t seem to be the cause. Unless (very likely) I’m 
>>>>>>>>> missing something. Anything else I can check?
>>>>>>>>> 
>>>>>>>>> Regards,
>>>>>>>>> 
>>>>>>>>> Pim
>>>>>>>>> 
>>>>>>>>>> On 08 Dec 2014, at 17:03, Ralph Castain <r...@open-mpi.org> wrote:
>>>>>>>>>> 
>>>>>>>>>> As Brice said, OMPI has its own embedded version of hwloc that we 
>>>>>>>>>> use, so there is no Slurm interaction to be considered. The most 
>>>>>>>>>> likely cause is that one or more of your nodes is picking up a 
>>>>>>>>>> different version of OMPI. So things “work” if you happen to get 
>>>>>>>>>> nodes where all the versions match, and “fail” when you get a 
>>>>>>>>>> combination that includes a different version.
>>>>>>>>>> 
>>>>>>>>>> Is there some way you can narrow down your search to find the 
>>>>>>>>>> node(s) that are picking up the different version?
>>>>>>>>>> 
>>>>>>>>>> 
>>>>>>>>>>> On Dec 8, 2014, at 7:48 AM, Pim Schellart <p.schell...@gmail.com> 
>>>>>>>>>>> wrote:
>>>>>>>>>>> 
>>>>>>>>>>> Dear Brice,
>>>>>>>>>>> 
>>>>>>>>>>> I am not sure why this is happening since all code seems to be 
>>>>>>>>>>> using the same hwloc library version (1.8) but it does :) An MPI 
>>>>>>>>>>> program is started through SLURM on two nodes with four CPU cores 
>>>>>>>>>>> total (divided over the nodes) using the following script:
>>>>>>>>>>> 
>>>>>>>>>>> #! /bin/bash
>>>>>>>>>>> #SBATCH -N 2 -n 4
>>>>>>>>>>> /usr/bin/mpiexec /usr/bin/lstopo --version
>>>>>>>>>>> /usr/bin/mpiexec /usr/bin/lstopo --of xml
>>>>>>>>>>> /usr/bin/mpiexec  /path/to/my_mpi_code
>>>>>>>>>>> 
>>>>>>>>>>> When this is submitted multiple times it gives “out-of-order” 
>>>>>>>>>>> warnings in about 9/10 cases but works without warnings in 1/10 
>>>>>>>>>>> cases. I attached the output (with xml) for both the working and 
>>>>>>>>>>> `broken` case. Note that the xml is of course printed (differently) 
>>>>>>>>>>> multiple times for each task/core. As always, any help would be 
>>>>>>>>>>> appreciated.
>>>>>>>>>>> 
>>>>>>>>>>> Regards,
>>>>>>>>>>> 
>>>>>>>>>>> Pim Schellart
>>>>>>>>>>> 
>>>>>>>>>>> P.S. $ mpirun --version
>>>>>>>>>>> mpirun (Open MPI) 1.6.5
>>>>>>>>>>> 
>>>>>>>>>>> <broken.log><working.log>
>>>>>>>>>>> 
>>>>>>>>>>>> On 07 Dec 2014, at 13:50, Brice Goglin <brice.gog...@inria.fr> 
>>>>>>>>>>>> wrote:
>>>>>>>>>>>> 
>>>>>>>>>>>> Hello
>>>>>>>>>>>> The github issue you're refering to was closed 18 months ago. The
>>>>>>>>>>>> warning (it's not an error) is only supposed to appear if you're
>>>>>>>>>>>> importing in a recent hwloc a XML that was exported from a old 
>>>>>>>>>>>> hwloc. I
>>>>>>>>>>>> don't see how that could happen when using Open MPI since the hwloc
>>>>>>>>>>>> versions on both sides is the same.
>>>>>>>>>>>> Make sure you're not confusing with another error described here
>>>>>>>>>>>> 
>>>>>>>>>>>> http://www.open-mpi.org/projects/hwloc/doc/v1.10.0/a00028.php#faq_os_error
>>>>>>>>>>>> Otherwise please report the exact Open MPI and/or hwloc versions 
>>>>>>>>>>>> as well
>>>>>>>>>>>> as the XML lstopo output on the nodes that raise the warning 
>>>>>>>>>>>> (lstopo
>>>>>>>>>>>> foo.xml). Send these to hwloc mailing lists such as
>>>>>>>>>>>> hwloc-us...@open-mpi.org or hwloc-de...@open-mpi.org
>>>>>>>>>>>> Thanks
>>>>>>>>>>>> Brice
>>>>>>>>>>>> 
>>>>>>>>>>>> 
>>>>>>>>>>>> Le 07/12/2014 13:29, Pim Schellart a écrit :
>>>>>>>>>>>>> Dear OpenMPI developers,
>>>>>>>>>>>>> 
>>>>>>>>>>>>> this might be a bit off topic but when using the SLURM scheduler 
>>>>>>>>>>>>> (with cpuset support) on Ubuntu 14.04 (openmpi 1.6) hwloc 
>>>>>>>>>>>>> sometimes gives a "out-of-order topology discovery” error. 
>>>>>>>>>>>>> According to issue #103 on github 
>>>>>>>>>>>>> (https://github.com/open-mpi/hwloc/issues/103) this error was 
>>>>>>>>>>>>> discussed before and it was possible to sort it out in 
>>>>>>>>>>>>> “insert_object_by_parent”, is this still considered? If not, what 
>>>>>>>>>>>>> (top level) hwloc API call should we look for in the SLURM 
>>>>>>>>>>>>> sources to start debugging? Any help will be most welcome.
>>>>>>>>>>>>> 
>>>>>>>>>>>>> Kind regards,
>>>>>>>>>>>>> 
>>>>>>>>>>>>> Pim Schellart
>>>>>>>>>>>>> _______________________________________________
>>>>>>>>>>>>> devel mailing list
>>>>>>>>>>>>> de...@open-mpi.org
>>>>>>>>>>>>> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
>>>>>>>>>>>>> Link to this post: 
>>>>>>>>>>>>> http://www.open-mpi.org/community/lists/devel/2014/12/16441.php
>>>>>>>>>>> _______________________________________________
>>>>>>>>>>> devel mailing list
>>>>>>>>>>> de...@open-mpi.org
>>>>>>>>>>> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
>>>>>>>>>>> Link to this post: 
>>>>>>>>>>> http://www.open-mpi.org/community/lists/devel/2014/12/16447.php
>>>>>>>>>> _______________________________________________
>>>>>>>>>> devel mailing list
>>>>>>>>>> de...@open-mpi.org
>>>>>>>>>> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
>>>>>>>>>> Link to this post: 
>>>>>>>>>> http://www.open-mpi.org/community/lists/devel/2014/12/16448.php
>>>>>>>>> _______________________________________________
>>>>>>>>> devel mailing list
>>>>>>>>> de...@open-mpi.org
>>>>>>>>> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
>>>>>>>>> Link to this post: 
>>>>>>>>> http://www.open-mpi.org/community/lists/devel/2014/12/16449.php
>>>>>>>> _______________________________________________
>>>>>>>> devel mailing list
>>>>>>>> de...@open-mpi.org
>>>>>>>> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
>>>>>>>> Link to this post: 
>>>>>>>> http://www.open-mpi.org/community/lists/devel/2014/12/16450.php
>>>>>>> _______________________________________________
>>>>>>> devel mailing list
>>>>>>> de...@open-mpi.org
>>>>>>> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
>>>>>>> Link to this post: 
>>>>>>> http://www.open-mpi.org/community/lists/devel/2014/12/16451.php
>>>>>> _______________________________________________
>>>>>> devel mailing list
>>>>>> de...@open-mpi.org
>>>>>> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
>>>>>> Link to this post: 
>>>>>> http://www.open-mpi.org/community/lists/devel/2014/12/16452.php
>>>>> _______________________________________________
>>>>> devel mailing list
>>>>> de...@open-mpi.org
>>>>> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
>>>>> Link to this post: 
>>>>> http://www.open-mpi.org/community/lists/devel/2014/12/16453.php
>>>> _______________________________________________
>>>> devel mailing list
>>>> de...@open-mpi.org
>>>> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
>>>> Link to this post: 
>>>> http://www.open-mpi.org/community/lists/devel/2014/12/16458.php
>>> _______________________________________________
>>> devel mailing list
>>> de...@open-mpi.org
>>> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
>>> Link to this post: 
>>> http://www.open-mpi.org/community/lists/devel/2014/12/16460.php
>> _______________________________________________
>> devel mailing list
>> de...@open-mpi.org
>> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
>> Link to this post: 
>> http://www.open-mpi.org/community/lists/devel/2014/12/16464.php
> 
> _______________________________________________
> devel mailing list
> de...@open-mpi.org
> Subscription: http://www.open-mpi.org/mailman/listinfo.cgi/devel
> Link to this post: 
> http://www.open-mpi.org/community/lists/devel/2014/12/16465.php